package replication import "testing" func TestPhase14_BoundaryTruthsStayDistinct(t *testing.T) { core := NewCoreEngine() core.ApplyEvent(CommittedLSNAdvanced{ID: "vol-boundary", CommittedLSN: 20}) core.ApplyEvent(CheckpointAdvanced{ID: "vol-boundary", CheckpointLSN: 12}) core.ApplyEvent(DiagnosticShippedAdvanced{ID: "vol-boundary", ShippedLSN: 30}) result := core.ApplyEvent(BarrierAccepted{ID: "vol-boundary", FlushedLSN: 8}) if result.Projection.Boundary.CommittedLSN != 20 { t.Fatalf("committed_lsn=%d", result.Projection.Boundary.CommittedLSN) } if result.Projection.Boundary.CheckpointLSN != 12 { t.Fatalf("checkpoint_lsn=%d", result.Projection.Boundary.CheckpointLSN) } if result.Projection.Boundary.DurableLSN != 8 { t.Fatalf("durable_lsn=%d", result.Projection.Boundary.DurableLSN) } if result.Projection.Boundary.DiagnosticShippedLSN != 30 { t.Fatalf("diagnostic_shipped_lsn=%d", result.Projection.Boundary.DiagnosticShippedLSN) } } func TestPhase14_RecoveryCatchUpBlocksReplicaReadyOverclaim(t *testing.T) { core := NewCoreEngine() core.ApplyEvent(AssignmentDelivered{ ID: "vol-recovery", Epoch: 4, Role: RoleReplica, Replicas: []ReplicaAssignment{ {ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.20:9333", CtrlAddr: "10.0.0.20:9334", Version: 1}}, }, }) core.ApplyEvent(RoleApplied{ID: "vol-recovery"}) core.ApplyEvent(ReceiverReadyObserved{ID: "vol-recovery"}) result := core.ApplyEvent(CatchUpPlanned{ID: "vol-recovery", TargetLSN: 40}) if result.Projection.Mode.Name != ModeBootstrapPending { t.Fatalf("mode=%s", result.Projection.Mode.Name) } if result.Projection.Publication.Reason != "recovery_in_progress" { t.Fatalf("publication_reason=%q", result.Projection.Publication.Reason) } if result.State.Recovery.Phase != RecoveryCatchingUp { t.Fatalf("recovery_phase=%s", result.State.Recovery.Phase) } if result.Projection.Recovery.Phase != RecoveryCatchingUp { t.Fatalf("projection_recovery_phase=%s", result.Projection.Recovery.Phase) } if result.Projection.Boundary.TargetLSN != 40 { t.Fatalf("target_lsn=%d", result.Projection.Boundary.TargetLSN) } result = core.ApplyEvent(RecoveryProgressObserved{ID: "vol-recovery", AchievedLSN: 25}) if result.Projection.Mode.Name != ModeBootstrapPending { t.Fatalf("mode=%s", result.Projection.Mode.Name) } if result.Projection.Publication.Healthy { t.Fatal("catch-up progress must not establish healthy publication") } if result.Projection.Boundary.AchievedLSN != 25 { t.Fatalf("achieved_lsn=%d", result.Projection.Boundary.AchievedLSN) } result = core.ApplyEvent(CatchUpCompleted{ID: "vol-recovery", AchievedLSN: 40}) if result.State.Recovery.Phase != RecoveryIdle { t.Fatalf("recovery_phase=%s", result.State.Recovery.Phase) } if result.Projection.Recovery.AchievedLSN != 40 { t.Fatalf("projection_achieved_lsn=%d", result.Projection.Recovery.AchievedLSN) } if result.Projection.Boundary.DurableLSN != 40 { t.Fatalf("durable_lsn=%d", result.Projection.Boundary.DurableLSN) } if result.Projection.Mode.Name != ModeReplicaReady { t.Fatalf("mode=%s", result.Projection.Mode.Name) } if result.Projection.Publication.Healthy { t.Fatal("catch-up completion on replica must not overclaim publish healthy") } } func TestPhase14_RebuildCommitClosesRecoveryBoundary(t *testing.T) { core := NewCoreEngine() core.ApplyEvent(AssignmentDelivered{ ID: "vol-rebuild", Epoch: 8, Role: RoleReplica, Replicas: []ReplicaAssignment{ {ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.21:9333", CtrlAddr: "10.0.0.21:9334", Version: 1}}, }, }) core.ApplyEvent(RoleApplied{ID: "vol-rebuild"}) core.ApplyEvent(ReceiverReadyObserved{ID: "vol-rebuild"}) result := core.ApplyEvent(NeedsRebuildObserved{ID: "vol-rebuild", Reason: "gap_too_large"}) if result.State.Recovery.Phase != RecoveryNeedsRebuild { t.Fatalf("recovery_phase=%s", result.State.Recovery.Phase) } if result.Projection.Mode.Name != ModeNeedsRebuild { t.Fatalf("mode=%s", result.Projection.Mode.Name) } result = core.ApplyEvent(RebuildStarted{ID: "vol-rebuild", TargetLSN: 64}) if result.State.Recovery.Phase != RecoveryRebuilding { t.Fatalf("recovery_phase=%s", result.State.Recovery.Phase) } if result.Projection.Recovery.Phase != RecoveryRebuilding { t.Fatalf("projection_recovery_phase=%s", result.Projection.Recovery.Phase) } if result.Projection.Boundary.TargetLSN != 64 { t.Fatalf("target_lsn=%d", result.Projection.Boundary.TargetLSN) } if result.Projection.Mode.Name != ModeNeedsRebuild { t.Fatalf("mode=%s", result.Projection.Mode.Name) } result = core.ApplyEvent(RebuildCommitted{ ID: "vol-rebuild", AchievedLSN: 64, FlushedLSN: 64, CheckpointLSN: 64, }) if result.State.Recovery.Phase != RecoveryIdle { t.Fatalf("recovery_phase=%s", result.State.Recovery.Phase) } if result.Projection.Recovery.Phase != RecoveryIdle { t.Fatalf("projection_recovery_phase=%s", result.Projection.Recovery.Phase) } if result.Projection.Boundary.AchievedLSN != 64 { t.Fatalf("achieved_lsn=%d", result.Projection.Boundary.AchievedLSN) } if result.Projection.Boundary.DurableLSN != 64 { t.Fatalf("durable_lsn=%d", result.Projection.Boundary.DurableLSN) } if result.Projection.Boundary.CheckpointLSN != 64 { t.Fatalf("checkpoint_lsn=%d", result.Projection.Boundary.CheckpointLSN) } if result.Projection.Mode.Name != ModeReplicaReady { t.Fatalf("mode=%s", result.Projection.Mode.Name) } if result.Projection.Publication.Healthy { t.Fatal("replica rebuild completion must not overclaim healthy publication") } } func TestPhase14_AssignmentChangeClearsStaleRecoveryTruth(t *testing.T) { core := NewCoreEngine() core.ApplyEvent(AssignmentDelivered{ ID: "vol-reassign", Epoch: 1, Role: RoleReplica, Replicas: []ReplicaAssignment{ {ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.30:9333", CtrlAddr: "10.0.0.30:9334", Version: 1}}, }, }) core.ApplyEvent(RoleApplied{ID: "vol-reassign"}) core.ApplyEvent(ReceiverReadyObserved{ID: "vol-reassign"}) core.ApplyEvent(CatchUpPlanned{ID: "vol-reassign", TargetLSN: 70}) core.ApplyEvent(RecoveryProgressObserved{ID: "vol-reassign", AchievedLSN: 41}) result := core.ApplyEvent(AssignmentDelivered{ ID: "vol-reassign", Epoch: 2, Role: RoleReplica, Replicas: []ReplicaAssignment{ {ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.31:9333", CtrlAddr: "10.0.0.31:9334", Version: 2}}, }, }) if result.State.Recovery.Phase != RecoveryIdle { t.Fatalf("recovery_phase=%s", result.State.Recovery.Phase) } if result.Projection.Boundary.TargetLSN != 0 { t.Fatalf("target_lsn=%d", result.Projection.Boundary.TargetLSN) } if result.Projection.Boundary.AchievedLSN != 0 { t.Fatalf("achieved_lsn=%d", result.Projection.Boundary.AchievedLSN) } if result.Projection.Mode.Name != ModeBootstrapPending { t.Fatalf("mode=%s", result.Projection.Mode.Name) } if result.Projection.Publication.Reason != "awaiting_role_apply" { t.Fatalf("publication_reason=%q", result.Projection.Publication.Reason) } } func TestPhase14_MultiReplicaCatchUpAggregatesUntilAllComplete(t *testing.T) { core := NewCoreEngine() core.ApplyEvent(AssignmentDelivered{ ID: "vol-multi-catchup", Epoch: 1, Role: RolePrimary, RecoveryTarget: SessionCatchUp, Replicas: []ReplicaAssignment{ {ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.40:9333", CtrlAddr: "10.0.0.40:9334", Version: 1}}, {ReplicaID: "replica-2", Endpoint: Endpoint{DataAddr: "10.0.0.41:9333", CtrlAddr: "10.0.0.41:9334", Version: 1}}, }, }) core.ApplyEvent(RoleApplied{ID: "vol-multi-catchup"}) core.ApplyEvent(ShipperConfiguredObserved{ID: "vol-multi-catchup"}) core.ApplyEvent(ShipperConnectedObserved{ID: "vol-multi-catchup"}) result := core.ApplyEvent(CatchUpPlanned{ID: "vol-multi-catchup", ReplicaID: "replica-1", TargetLSN: 100}) if result.State.Recovery.Phase != RecoveryCatchingUp { t.Fatalf("phase_after_first_plan=%s", result.State.Recovery.Phase) } result = core.ApplyEvent(CatchUpPlanned{ID: "vol-multi-catchup", ReplicaID: "replica-2", TargetLSN: 100}) if result.State.Recovery.Phase != RecoveryCatchingUp { t.Fatalf("phase_after_second_plan=%s", result.State.Recovery.Phase) } result = core.ApplyEvent(RecoveryProgressObserved{ID: "vol-multi-catchup", ReplicaID: "replica-1", AchievedLSN: 100}) if result.Projection.Recovery.AchievedLSN != 0 { t.Fatalf("aggregate_achieved_lsn=%d", result.Projection.Recovery.AchievedLSN) } result = core.ApplyEvent(CatchUpCompleted{ID: "vol-multi-catchup", ReplicaID: "replica-1", AchievedLSN: 100}) if result.State.Recovery.Phase != RecoveryCatchingUp { t.Fatalf("phase_after_first_complete=%s", result.State.Recovery.Phase) } if result.Projection.Boundary.DurableLSN != 0 { t.Fatalf("durable_lsn_after_first_complete=%d", result.Projection.Boundary.DurableLSN) } if result.Projection.Publication.Healthy { t.Fatal("publication should not become healthy before all replicas complete") } result = core.ApplyEvent(CatchUpCompleted{ID: "vol-multi-catchup", ReplicaID: "replica-2", AchievedLSN: 100}) if result.State.Recovery.Phase != RecoveryIdle { t.Fatalf("phase_after_all_complete=%s", result.State.Recovery.Phase) } if result.Projection.Boundary.DurableLSN != 100 { t.Fatalf("durable_lsn_after_all_complete=%d", result.Projection.Boundary.DurableLSN) } if result.Projection.Mode.Name != ModePublishHealthy { t.Fatalf("mode_after_all_complete=%s", result.Projection.Mode.Name) } if !result.Projection.Publication.Healthy { t.Fatal("publication should become healthy after all replicas complete") } }