package v2bridge import ( "bytes" "net" "path/filepath" "strings" "testing" bridge "github.com/seaweedfs/seaweedfs/sw-block/bridge/blockvol" engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" ) // ============================================================ // Phase 09 P1: Full-base execution closure // // Proofs: // 1. Component: TCP transfer + local install (bridge level) // 2. One-chain: engine plan → RebuildExecutor → v2bridge → blockvol install → completion // 3. Fail-closed: connection refused, epoch mismatch, partial transfer, no address // ============================================================ // createTestVolNamed creates a real file-backed BlockVol in the given dir. func createTestVolNamed(t *testing.T, dir, name string) *blockvol.BlockVol { t.Helper() path := filepath.Join(dir, name) v, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{ VolumeSize: 1 * 1024 * 1024, BlockSize: 4096, WALSize: 256 * 1024, }) if err != nil { t.Fatalf("CreateBlockVol %s: %v", name, err) } return v } // verifyLBAMatch reads LBAs from both vols and verifies they match. func verifyLBAMatch(t *testing.T, primaryVol, replicaVol *blockvol.BlockVol, lbaCount int) { t.Helper() blockSize := primaryVol.Info().BlockSize for i := 0; i < lbaCount; i++ { pdata, perr := primaryVol.ReadLBA(uint64(i), blockSize) rdata, rerr := replicaVol.ReadLBA(uint64(i), blockSize) if perr != nil { t.Fatalf("primary ReadLBA(%d): %v", i, perr) } if rerr != nil { t.Fatalf("replica ReadLBA(%d): %v", i, rerr) } if !bytes.Equal(pdata, rdata) { t.Fatalf("LBA %d mismatch: primary[0]=%d replica[0]=%d", i, pdata[0], rdata[0]) } } } // --- Component Proof: TCP transfer + local install --- func TestP1_TransferFullBase_RealTCP(t *testing.T) { dir := t.TempDir() // Primary: write data + flush to populate extent region. primaryVol := createTestVolNamed(t, dir, "primary.blockvol") defer primaryVol.Close() for i := 0; i < 20; i++ { primaryVol.WriteLBA(uint64(i), makeBlock(byte('A'+i%26))) } primaryVol.ForceFlush() primaryState := NewReader(primaryVol).ReadState() t.Logf("primary: head=%d tail=%d committed=%d checkpoint=%d", primaryState.WALHeadLSN, primaryState.WALTailLSN, primaryState.CommittedLSN, primaryState.CheckpointLSN) // Start rebuild server on primary (existing V1 code). rebuildServer, err := blockvol.NewRebuildServer(primaryVol, "127.0.0.1:0") if err != nil { t.Fatalf("NewRebuildServer: %v", err) } rebuildServer.Serve() defer rebuildServer.Stop() t.Logf("rebuild server on %s", rebuildServer.Addr()) // Replica: empty vol, same geometry. replicaVol := createTestVolNamed(t, dir, "replica.blockvol") defer replicaVol.Close() // Create executor for replica, pointing to primary's rebuild server. executor := NewExecutor(replicaVol, rebuildServer.Addr()) // Transfer full base. if _, err := executor.TransferFullBase(primaryState.CommittedLSN); err != nil { t.Fatalf("TransferFullBase: %v", err) } // Verify: replica LBA data matches primary (reads from extent since // replica has no WAL entries or dirty map entries). verifyLBAMatch(t, primaryVol, replicaVol, 20) t.Log("P1 component proof: TCP transfer + local install verified — LBA data matches") } // --- One-Chain Proof: engine → executor → bridge → blockvol → completion --- // untrustedReaderShim wraps a Reader but reports CheckpointTrusted=false. // This forces the engine's RebuildSourceDecision to select RebuildFullBase // instead of RebuildSnapshotTail, so the one-chain test exercises // TransferFullBase specifically. type untrustedReaderShim struct{ r *Reader } func (s *untrustedReaderShim) ReadState() bridge.BlockVolState { rs := s.r.ReadState() return bridge.BlockVolState{ WALHeadLSN: rs.WALHeadLSN, WALTailLSN: rs.WALTailLSN, CommittedLSN: rs.CommittedLSN, CheckpointLSN: rs.CheckpointLSN, CheckpointTrusted: false, // force full-base path } } func TestP1_FullBaseRebuild_OneChain(t *testing.T) { dir := t.TempDir() // Primary: write data + flush → force rebuild condition. primaryVol := createTestVolNamed(t, dir, "primary.blockvol") defer primaryVol.Close() for i := 0; i < 20; i++ { primaryVol.WriteLBA(uint64(i), makeBlock(byte('A'+i%26))) } primaryVol.ForceFlush() primaryState := NewReader(primaryVol).ReadState() if primaryState.WALTailLSN == 0 { t.Fatal("ForceFlush must advance tail for rebuild condition") } // Start rebuild server on primary. rebuildServer, err := blockvol.NewRebuildServer(primaryVol, "127.0.0.1:0") if err != nil { t.Fatalf("NewRebuildServer: %v", err) } rebuildServer.Serve() defer rebuildServer.Stop() // Replica: empty vol. replicaVol := createTestVolNamed(t, dir, "replica.blockvol") defer replicaVol.Close() // Engine setup: StorageAdapter reads from PRIMARY (for planning). // Use untrustedReaderShim to force full-base rebuild path so this // test exercises TransferFullBase specifically (not TransferSnapshot). primaryReader := NewReader(primaryVol) primaryPinner := NewPinner(primaryVol) sa := bridge.NewStorageAdapter( &untrustedReaderShim{primaryReader}, &pinnerShim{primaryPinner}, ) ca := bridge.NewControlAdapter() driver := engine.NewRecoveryDriver(sa) // Step 1: assignment — register the replica sender. intent := ca.ToAssignmentIntent( bridge.MasterAssignment{VolumeName: "vol1", Epoch: 1, Role: "primary"}, []bridge.MasterAssignment{ {VolumeName: "vol1", ReplicaServerID: "vs2", Role: "replica", DataAddr: "10.0.0.2:9333", CtrlAddr: "10.0.0.2:9334"}, }, ) driver.Orchestrator.ProcessAssignment(intent) // Step 2: plan recovery — replicaLSN=0 with tail>0 forces NeedsRebuild. plan, err := driver.PlanRecovery("vol1/vs2", 0) if err != nil { t.Fatalf("PlanRecovery: %v", err) } if plan.Outcome != engine.OutcomeNeedsRebuild { t.Fatalf("outcome=%s, want NeedsRebuild", plan.Outcome) } // Step 3: rebuild assignment — switch sender to rebuild session. rebuildIntent := ca.ToAssignmentIntent( bridge.MasterAssignment{VolumeName: "vol1", Epoch: 1, Role: "primary"}, []bridge.MasterAssignment{ {VolumeName: "vol1", ReplicaServerID: "vs2", Role: "rebuilding", DataAddr: "10.0.0.2:9333", CtrlAddr: "10.0.0.2:9334"}, }, ) driver.Orchestrator.ProcessAssignment(rebuildIntent) // Step 4: plan rebuild from real storage. rebuildPlan, err := driver.PlanRebuild("vol1/vs2") if err != nil { t.Fatalf("PlanRebuild: %v", err) } if rebuildPlan.RebuildSource != engine.RebuildFullBase { t.Fatalf("source=%s, want full_base (untrusted checkpoint should force this)", rebuildPlan.RebuildSource) } t.Logf("rebuild plan: source=%s target=%d", rebuildPlan.RebuildSource, rebuildPlan.RebuildTargetLSN) // Step 5: RebuildExecutor with real IO wired to v2bridge executor (on replica vol). replicaExecutor := NewExecutor(replicaVol, rebuildServer.Addr()) exec := engine.NewRebuildExecutor(driver, rebuildPlan) exec.IO = replicaExecutor if err := exec.Execute(); err != nil { t.Fatalf("RebuildExecutor.Execute: %v", err) } // Step 6: verify sender state → InSync. s := driver.Orchestrator.Registry.Sender("vol1/vs2") if s.State() != engine.StateInSync { t.Fatalf("state=%s, want InSync", s.State()) } // Step 7: verify pins released. if primaryPinner.ActiveHoldCount() != 0 { t.Fatalf("%d pins leaked", primaryPinner.ActiveHoldCount()) } // Step 8: verify LBA data matches. verifyLBAMatch(t, primaryVol, replicaVol, 20) // Step 9: verify observability — execution log shows rebuild events. events := driver.Orchestrator.Log.EventsFor("vol1/vs2") hasStarted := false hasCompleted := false for _, ev := range events { if ev.Event == "exec_rebuild_started" { hasStarted = true } if ev.Event == "exec_rebuild_completed" { hasCompleted = true } } if !hasStarted || !hasCompleted { t.Fatalf("observability: started=%v completed=%v", hasStarted, hasCompleted) } t.Log("P1 one-chain: plan(full_base) → RebuildExecutor(IO=v2bridge) → TCP → local install → InSync → pins released → data verified") } // --- Non-empty replica: stale state must be cleared --- func TestP1_TransferFullBase_NonEmptyReplica(t *testing.T) { dir := t.TempDir() // Primary: write data + flush. primaryVol := createTestVolNamed(t, dir, "primary.blockvol") defer primaryVol.Close() for i := 0; i < 10; i++ { primaryVol.WriteLBA(uint64(i), makeBlock(byte('P'))) } primaryVol.ForceFlush() rebuildServer, err := blockvol.NewRebuildServer(primaryVol, "127.0.0.1:0") if err != nil { t.Fatalf("NewRebuildServer: %v", err) } rebuildServer.Serve() defer rebuildServer.Stop() // Replica: has STALE data + WAL entries (simulates a previously-used replica). replicaVol := createTestVolNamed(t, dir, "replica.blockvol") defer replicaVol.Close() for i := 0; i < 5; i++ { replicaVol.WriteLBA(uint64(i), makeBlock(byte('R'))) } // Confirm replica has WAL entries and dirty map entries. replicaStateBefore := NewReader(replicaVol).ReadState() if replicaStateBefore.WALHeadLSN == 0 { t.Fatal("replica must have WAL entries before rebuild") } t.Logf("replica before: head=%d", replicaStateBefore.WALHeadLSN) // Transfer full base — must clear stale state. executor := NewExecutor(replicaVol, rebuildServer.Addr()) if _, err := executor.TransferFullBase(0); err != nil { t.Fatalf("TransferFullBase: %v", err) } // Verify: replica reads primary's data (not stale 'R' blocks). for i := 0; i < 10; i++ { data, err := replicaVol.ReadLBA(uint64(i), replicaVol.Info().BlockSize) if err != nil { t.Fatalf("ReadLBA(%d): %v", i, err) } if data[0] != 'P' { t.Fatalf("LBA %d: got %c, want P (stale data not cleared)", i, data[0]) } } // Verify: WAL state was reset (no stale entries overlaying the new extent). replicaStateAfter := NewReader(replicaVol).ReadState() t.Logf("replica after: head=%d tail=%d checkpoint=%d", replicaStateAfter.WALHeadLSN, replicaStateAfter.WALTailLSN, replicaStateAfter.CheckpointLSN) t.Log("P1 non-empty replica: stale WAL/dirty state cleared, primary data installed correctly") } // --- Pre-flush correctness: unflushed WAL entries are in the extent --- func TestP1_TransferFullBase_UnflushedEntries(t *testing.T) { dir := t.TempDir() // Primary: write data, flush SOME, then write MORE that stay in WAL. primaryVol := createTestVolNamed(t, dir, "primary.blockvol") defer primaryVol.Close() for i := 0; i < 10; i++ { primaryVol.WriteLBA(uint64(i), makeBlock(byte('A'+i%26))) } primaryVol.ForceFlush() // These 5 writes are in the WAL, NOT yet flushed to extent. for i := 10; i < 15; i++ { primaryVol.WriteLBA(uint64(i), makeBlock(byte('A'+i%26))) } primaryState := NewReader(primaryVol).ReadState() t.Logf("primary: head=%d tail=%d committed=%d checkpoint=%d", primaryState.WALHeadLSN, primaryState.WALTailLSN, primaryState.CommittedLSN, primaryState.CheckpointLSN) // Confirm: checkpoint < head (unflushed entries exist). if primaryState.CheckpointLSN >= primaryState.WALHeadLSN { t.Fatal("need unflushed entries: checkpoint must be < head") } rebuildServer, err := blockvol.NewRebuildServer(primaryVol, "127.0.0.1:0") if err != nil { t.Fatalf("NewRebuildServer: %v", err) } rebuildServer.Serve() defer rebuildServer.Stop() // Replica: empty vol. replicaVol := createTestVolNamed(t, dir, "replica.blockvol") defer replicaVol.Close() executor := NewExecutor(replicaVol, rebuildServer.Addr()) if _, err := executor.TransferFullBase(primaryState.CommittedLSN); err != nil { t.Fatalf("TransferFullBase: %v", err) } // Verify: ALL 15 LBAs match — including the 5 that were unflushed. // The rebuild server's pre-flush ensures they are in the extent. verifyLBAMatch(t, primaryVol, replicaVol, 15) t.Log("P1 pre-flush: unflushed WAL entries flushed by rebuild server before extent copy — all data correct") } // --- Convergence proof: achievedLSN > targetLSN, no split truth --- func TestP1_FullBaseRebuild_AchievedConvergence(t *testing.T) { dir := t.TempDir() // Primary: write initial data + flush. primaryVol := createTestVolNamed(t, dir, "primary.blockvol") defer primaryVol.Close() for i := 0; i < 20; i++ { primaryVol.WriteLBA(uint64(i), makeBlock(byte('A'+i%26))) } primaryVol.ForceFlush() primaryState := NewReader(primaryVol).ReadState() if primaryState.WALTailLSN == 0 { t.Fatal("ForceFlush must advance tail for rebuild condition") } rebuildServer, err := blockvol.NewRebuildServer(primaryVol, "127.0.0.1:0") if err != nil { t.Fatalf("NewRebuildServer: %v", err) } rebuildServer.Serve() defer rebuildServer.Stop() // Replica: empty vol. replicaVol := createTestVolNamed(t, dir, "replica.blockvol") defer replicaVol.Close() // Engine setup with untrusted reader to force full-base path. primaryReader := NewReader(primaryVol) primaryPinner := NewPinner(primaryVol) sa := bridge.NewStorageAdapter( &untrustedReaderShim{primaryReader}, &pinnerShim{primaryPinner}, ) ca := bridge.NewControlAdapter() driver := engine.NewRecoveryDriver(sa) // Assignment + plan. intent := ca.ToAssignmentIntent( bridge.MasterAssignment{VolumeName: "vol1", Epoch: 1, Role: "primary"}, []bridge.MasterAssignment{ {VolumeName: "vol1", ReplicaServerID: "vs2", Role: "replica", DataAddr: "10.0.0.2:9333", CtrlAddr: "10.0.0.2:9334"}, }, ) driver.Orchestrator.ProcessAssignment(intent) plan, _ := driver.PlanRecovery("vol1/vs2", 0) if plan.Outcome != engine.OutcomeNeedsRebuild { t.Fatalf("outcome=%s", plan.Outcome) } rebuildIntent := ca.ToAssignmentIntent( bridge.MasterAssignment{VolumeName: "vol1", Epoch: 1, Role: "primary"}, []bridge.MasterAssignment{ {VolumeName: "vol1", ReplicaServerID: "vs2", Role: "rebuilding", DataAddr: "10.0.0.2:9333", CtrlAddr: "10.0.0.2:9334"}, }, ) driver.Orchestrator.ProcessAssignment(rebuildIntent) rebuildPlan, err := driver.PlanRebuild("vol1/vs2") if err != nil { t.Fatalf("PlanRebuild: %v", err) } targetLSN := rebuildPlan.RebuildTargetLSN t.Logf("plan: target=%d source=%s", targetLSN, rebuildPlan.RebuildSource) // --- Force achievedLSN > targetLSN --- // Write additional data to primary AFTER planning. The rebuild server // will see these via ForceFlush and serve an extent newer than the plan. for i := 20; i < 25; i++ { primaryVol.WriteLBA(uint64(i), makeBlock(byte('X'))) } postPlanState := NewReader(primaryVol).ReadState() t.Logf("primary after extra writes: head=%d (plan target was %d)", postPlanState.WALHeadLSN, targetLSN) // Execute rebuild with real IO. replicaExecutor := NewExecutor(replicaVol, rebuildServer.Addr()) exec := engine.NewRebuildExecutor(driver, rebuildPlan) exec.IO = replicaExecutor if err := exec.Execute(); err != nil { t.Fatalf("Execute: %v", err) } // --- Full convergence verification --- replicaState := NewReader(replicaVol).ReadState() localAchieved := replicaState.WALHeadLSN localCheckpoint := replicaState.CheckpointLSN // 1. achievedLSN > targetLSN — primary advanced between plan and transfer. if localAchieved <= targetLSN { t.Fatalf("achievedLSN %d must be > targetLSN %d (primary wrote 5 more entries)", localAchieved, targetLSN) } t.Logf("achievedLSN=%d > targetLSN=%d — confirmed", localAchieved, targetLSN) // 2. Sender reached InSync. s := driver.Orchestrator.Registry.Sender("vol1/vs2") if s.State() != engine.StateInSync { t.Fatalf("state=%s, want InSync", s.State()) } // 3. No split truth: local checkpoint = local head = achievedLSN. if localCheckpoint != localAchieved { t.Fatalf("split truth: checkpoint=%d != achieved=%d", localCheckpoint, localAchieved) } // 4. Receiver progress aligned to achievedLSN. // In this test the receiver is nil (standalone replica vol), so // ReceivedLSN returns 0. The fix is verified structurally by // Commit + SyncReceiverProgress; production tests with live // receivers will exercise the full path. receiverLSN := replicaVol.ReceivedLSN() t.Logf("receiver progress: %d (0 = no active receiver in this test)", receiverLSN) // 5. All 25 LBAs match (original 20 + 5 written after plan). verifyLBAMatch(t, primaryVol, replicaVol, 25) // 6. Pins released. if primaryPinner.ActiveHoldCount() != 0 { t.Fatalf("%d pins leaked", primaryPinner.ActiveHoldCount()) } // 7. Engine log shows rebuild completion. events := driver.Orchestrator.Log.EventsFor("vol1/vs2") hasCompleted := false for _, ev := range events { if ev.Event == "exec_rebuild_completed" { hasCompleted = true } } if !hasCompleted { t.Fatal("missing exec_rebuild_completed event") } t.Logf("convergence: target=%d achieved=%d checkpoint=%d — single truth verified", targetLSN, localAchieved, localCheckpoint) } // --- Stale-higher convergence: replica had higher LSN than rebuilt boundary --- func TestP1_TransferFullBase_StaleHigherThanAchieved(t *testing.T) { dir := t.TempDir() // Primary: small amount of data (10 entries → achievedLSN ~10). primaryVol := createTestVolNamed(t, dir, "primary.blockvol") defer primaryVol.Close() for i := 0; i < 10; i++ { primaryVol.WriteLBA(uint64(i), makeBlock(byte('P'))) } primaryVol.ForceFlush() rebuildServer, err := blockvol.NewRebuildServer(primaryVol, "127.0.0.1:0") if err != nil { t.Fatalf("NewRebuildServer: %v", err) } rebuildServer.Serve() defer rebuildServer.Stop() // Replica: has MORE data than primary (30 entries → higher nextLSN/receivedLSN). // This simulates a stale replica that diverged (e.g., old primary that was demoted). replicaVol := createTestVolNamed(t, dir, "replica.blockvol") defer replicaVol.Close() for i := 0; i < 30; i++ { replicaVol.WriteLBA(uint64(i), makeBlock(byte('R'))) } // Start receiver so replRecv has a high receivedLSN. if err := replicaVol.StartReplicaReceiver("127.0.0.1:0", "127.0.0.1:0"); err != nil { t.Fatalf("StartReplicaReceiver: %v", err) } staleLSN := replicaVol.ReceivedLSN() staleState := NewReader(replicaVol).ReadState() t.Logf("replica before: head=%d receivedLSN=%d (higher than primary)", staleState.WALHeadLSN, staleLSN) if staleLSN <= 10 { t.Fatalf("replica receivedLSN %d must be > primary's 10 for this test", staleLSN) } // Transfer full base — must RESET (not just advance) to achieved boundary. executor := NewExecutor(replicaVol, rebuildServer.Addr()) achieved, err := executor.TransferFullBase(0) if err != nil { t.Fatalf("TransferFullBase: %v", err) } // The achieved boundary should match the primary (~10), NOT the stale (~30). postState := NewReader(replicaVol).ReadState() postReceivedLSN := replicaVol.ReceivedLSN() t.Logf("replica after: head=%d checkpoint=%d receivedLSN=%d achieved=%d", postState.WALHeadLSN, postState.CheckpointLSN, postReceivedLSN, achieved) // nextLSN (via WALHeadLSN) must be reset to achieved, not kept at stale higher value. if postState.WALHeadLSN != achieved { t.Fatalf("split truth: WALHeadLSN=%d != achieved=%d (stale higher value not reset)", postState.WALHeadLSN, achieved) } // receivedLSN must be reset to achieved, not kept at stale higher value. if postReceivedLSN != achieved { t.Fatalf("split truth: receivedLSN=%d != achieved=%d (stale higher value not reset)", postReceivedLSN, achieved) } // Data must be primary's, not stale replica's. for i := 0; i < 10; i++ { data, err := replicaVol.ReadLBA(uint64(i), replicaVol.Info().BlockSize) if err != nil { t.Fatalf("ReadLBA(%d): %v", i, err) } if data[0] != 'P' { t.Fatalf("LBA %d: got %c, want P", i, data[0]) } } t.Logf("stale-higher: staleRecv=%d staleHead=%d → achieved=%d receivedLSN=%d — reset, not max", staleLSN, staleState.WALHeadLSN, achieved, postReceivedLSN) } // --- Live receiver: receivedLSN convergence through active receiver --- func TestP1_TransferFullBase_LiveReceiverConvergence(t *testing.T) { dir := t.TempDir() // Primary: write data + flush. primaryVol := createTestVolNamed(t, dir, "primary.blockvol") defer primaryVol.Close() for i := 0; i < 20; i++ { primaryVol.WriteLBA(uint64(i), makeBlock(byte('A'+i%26))) } primaryVol.ForceFlush() rebuildServer, err := blockvol.NewRebuildServer(primaryVol, "127.0.0.1:0") if err != nil { t.Fatalf("NewRebuildServer: %v", err) } rebuildServer.Serve() defer rebuildServer.Stop() // Replica: has an ACTIVE receiver before rebuild (simulates a replica // that was previously receiving WAL entries and now needs a rebuild). replicaVol := createTestVolNamed(t, dir, "replica.blockvol") defer replicaVol.Close() // Write some stale data so the replica has a non-zero receivedLSN. for i := 0; i < 3; i++ { replicaVol.WriteLBA(uint64(i), makeBlock(byte('R'))) } // Start a real receiver on the replica via StartReplicaReceiver // (sets vol.replRecv so ReceivedLSN() returns a real value). if err := replicaVol.StartReplicaReceiver("127.0.0.1:0", "127.0.0.1:0"); err != nil { t.Fatalf("StartReplicaReceiver: %v", err) } staleReceivedLSN := replicaVol.ReceivedLSN() t.Logf("replica before rebuild: receivedLSN=%d", staleReceivedLSN) if staleReceivedLSN == 0 { t.Fatal("receiver must have non-zero receivedLSN before rebuild") } // Transfer full base — must align receiver progress. executor := NewExecutor(replicaVol, rebuildServer.Addr()) achieved, err := executor.TransferFullBase(0) if err != nil { t.Fatalf("TransferFullBase: %v", err) } // Verify: receivedLSN advanced to achievedLSN. postReceivedLSN := replicaVol.ReceivedLSN() if postReceivedLSN != achieved { t.Fatalf("receiver split truth: receivedLSN=%d != achieved=%d", postReceivedLSN, achieved) } // Verify: LBA data matches primary (not stale 'R' blocks). verifyLBAMatch(t, primaryVol, replicaVol, 20) t.Logf("live receiver convergence: stale=%d → achieved=%d, receivedLSN=%d — no split truth", staleReceivedLSN, achieved, postReceivedLSN) } // --- Fail-Closed: connection refused --- func TestP1_TransferFullBase_ConnectionRefused(t *testing.T) { dir := t.TempDir() replicaVol := createTestVolNamed(t, dir, "replica.blockvol") defer replicaVol.Close() // Point to an address where nothing is listening. executor := NewExecutor(replicaVol, "127.0.0.1:1") _, err := executor.TransferFullBase(100) if err == nil { t.Fatal("should fail on connection refused") } t.Logf("connection refused: %v", err) } // --- Fail-Closed: epoch mismatch --- func TestP1_TransferFullBase_EpochMismatch(t *testing.T) { dir := t.TempDir() // Primary with epoch 5. primaryVol := createTestVolNamed(t, dir, "primary.blockvol") defer primaryVol.Close() primaryVol.SetEpoch(5) primaryVol.WriteLBA(0, makeBlock('A')) primaryVol.ForceFlush() rebuildServer, err := blockvol.NewRebuildServer(primaryVol, "127.0.0.1:0") if err != nil { t.Fatalf("NewRebuildServer: %v", err) } rebuildServer.Serve() defer rebuildServer.Stop() // Replica with epoch 3 (stale). replicaVol := createTestVolNamed(t, dir, "replica.blockvol") defer replicaVol.Close() replicaVol.SetEpoch(3) executor := NewExecutor(replicaVol, rebuildServer.Addr()) _, err = executor.TransferFullBase(100) if err == nil { t.Fatal("should fail on epoch mismatch") } if !strings.Contains(err.Error(), "EPOCH_MISMATCH") { t.Fatalf("expected EPOCH_MISMATCH, got: %v", err) } t.Logf("epoch mismatch: %v", err) } // --- Fail-Closed: no rebuild address --- func TestP1_TransferFullBase_NoAddress(t *testing.T) { vol := createTestVol(t) defer vol.Close() executor := NewExecutor(vol, "") _, err := executor.TransferFullBase(100) if err == nil { t.Fatal("should fail without rebuild address") } t.Logf("no address: %v", err) } // --- Fail-Closed: partial transfer (server closes mid-stream) --- func TestP1_TransferFullBase_PartialTransfer(t *testing.T) { dir := t.TempDir() replicaVol := createTestVolNamed(t, dir, "replica.blockvol") defer replicaVol.Close() // Start a fake server that sends one extent chunk then closes. ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("listen: %v", err) } defer ln.Close() go func() { conn, err := ln.Accept() if err != nil { return } // Read the request frame (discard). blockvol.ReadFrame(conn) // Send one extent chunk. chunk := make([]byte, 4096) for i := range chunk { chunk[i] = 0xFF } blockvol.WriteFrame(conn, blockvol.MsgRebuildExtent, chunk) // Close abruptly — no MsgRebuildDone. conn.Close() }() executor := NewExecutor(replicaVol, ln.Addr().String()) _, err = executor.TransferFullBase(100) if err == nil { t.Fatal("should fail on partial transfer (connection closed before Done)") } t.Logf("partial transfer: %v", err) }