mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-20 13:30:46 +02:00
pingqiu
92c006eb29
Track catch-up observations per replica so the volume-level recovery view stays in catching_up until all bounded replicas complete. This preserves the current bounded semantics while removing an overclaim that would block later multi-replica startup ownership work. Made-with: Cursor
250 lines
9.5 KiB
Go
250 lines
9.5 KiB
Go
package replication
|
|
|
|
import "testing"
|
|
|
|
func TestPhase14_BoundaryTruthsStayDistinct(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(CommittedLSNAdvanced{ID: "vol-boundary", CommittedLSN: 20})
|
|
core.ApplyEvent(CheckpointAdvanced{ID: "vol-boundary", CheckpointLSN: 12})
|
|
core.ApplyEvent(DiagnosticShippedAdvanced{ID: "vol-boundary", ShippedLSN: 30})
|
|
result := core.ApplyEvent(BarrierAccepted{ID: "vol-boundary", FlushedLSN: 8})
|
|
|
|
if result.Projection.Boundary.CommittedLSN != 20 {
|
|
t.Fatalf("committed_lsn=%d", result.Projection.Boundary.CommittedLSN)
|
|
}
|
|
if result.Projection.Boundary.CheckpointLSN != 12 {
|
|
t.Fatalf("checkpoint_lsn=%d", result.Projection.Boundary.CheckpointLSN)
|
|
}
|
|
if result.Projection.Boundary.DurableLSN != 8 {
|
|
t.Fatalf("durable_lsn=%d", result.Projection.Boundary.DurableLSN)
|
|
}
|
|
if result.Projection.Boundary.DiagnosticShippedLSN != 30 {
|
|
t.Fatalf("diagnostic_shipped_lsn=%d", result.Projection.Boundary.DiagnosticShippedLSN)
|
|
}
|
|
}
|
|
|
|
func TestPhase14_RecoveryCatchUpBlocksReplicaReadyOverclaim(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-recovery",
|
|
Epoch: 4,
|
|
Role: RoleReplica,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.20:9333", CtrlAddr: "10.0.0.20:9334", Version: 1}},
|
|
},
|
|
})
|
|
core.ApplyEvent(RoleApplied{ID: "vol-recovery"})
|
|
core.ApplyEvent(ReceiverReadyObserved{ID: "vol-recovery"})
|
|
|
|
result := core.ApplyEvent(CatchUpPlanned{ID: "vol-recovery", TargetLSN: 40})
|
|
if result.Projection.Mode.Name != ModeBootstrapPending {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
if result.Projection.Publication.Reason != "recovery_in_progress" {
|
|
t.Fatalf("publication_reason=%q", result.Projection.Publication.Reason)
|
|
}
|
|
if result.State.Recovery.Phase != RecoveryCatchingUp {
|
|
t.Fatalf("recovery_phase=%s", result.State.Recovery.Phase)
|
|
}
|
|
if result.Projection.Recovery.Phase != RecoveryCatchingUp {
|
|
t.Fatalf("projection_recovery_phase=%s", result.Projection.Recovery.Phase)
|
|
}
|
|
if result.Projection.Boundary.TargetLSN != 40 {
|
|
t.Fatalf("target_lsn=%d", result.Projection.Boundary.TargetLSN)
|
|
}
|
|
|
|
result = core.ApplyEvent(RecoveryProgressObserved{ID: "vol-recovery", AchievedLSN: 25})
|
|
if result.Projection.Mode.Name != ModeBootstrapPending {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
if result.Projection.Publication.Healthy {
|
|
t.Fatal("catch-up progress must not establish healthy publication")
|
|
}
|
|
if result.Projection.Boundary.AchievedLSN != 25 {
|
|
t.Fatalf("achieved_lsn=%d", result.Projection.Boundary.AchievedLSN)
|
|
}
|
|
|
|
result = core.ApplyEvent(CatchUpCompleted{ID: "vol-recovery", AchievedLSN: 40})
|
|
if result.State.Recovery.Phase != RecoveryIdle {
|
|
t.Fatalf("recovery_phase=%s", result.State.Recovery.Phase)
|
|
}
|
|
if result.Projection.Recovery.AchievedLSN != 40 {
|
|
t.Fatalf("projection_achieved_lsn=%d", result.Projection.Recovery.AchievedLSN)
|
|
}
|
|
if result.Projection.Boundary.DurableLSN != 40 {
|
|
t.Fatalf("durable_lsn=%d", result.Projection.Boundary.DurableLSN)
|
|
}
|
|
if result.Projection.Mode.Name != ModeReplicaReady {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
if result.Projection.Publication.Healthy {
|
|
t.Fatal("catch-up completion on replica must not overclaim publish healthy")
|
|
}
|
|
}
|
|
|
|
func TestPhase14_RebuildCommitClosesRecoveryBoundary(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-rebuild",
|
|
Epoch: 8,
|
|
Role: RoleReplica,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.21:9333", CtrlAddr: "10.0.0.21:9334", Version: 1}},
|
|
},
|
|
})
|
|
core.ApplyEvent(RoleApplied{ID: "vol-rebuild"})
|
|
core.ApplyEvent(ReceiverReadyObserved{ID: "vol-rebuild"})
|
|
|
|
result := core.ApplyEvent(NeedsRebuildObserved{ID: "vol-rebuild", Reason: "gap_too_large"})
|
|
if result.State.Recovery.Phase != RecoveryNeedsRebuild {
|
|
t.Fatalf("recovery_phase=%s", result.State.Recovery.Phase)
|
|
}
|
|
if result.Projection.Mode.Name != ModeNeedsRebuild {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
|
|
result = core.ApplyEvent(RebuildStarted{ID: "vol-rebuild", TargetLSN: 64})
|
|
if result.State.Recovery.Phase != RecoveryRebuilding {
|
|
t.Fatalf("recovery_phase=%s", result.State.Recovery.Phase)
|
|
}
|
|
if result.Projection.Recovery.Phase != RecoveryRebuilding {
|
|
t.Fatalf("projection_recovery_phase=%s", result.Projection.Recovery.Phase)
|
|
}
|
|
if result.Projection.Boundary.TargetLSN != 64 {
|
|
t.Fatalf("target_lsn=%d", result.Projection.Boundary.TargetLSN)
|
|
}
|
|
if result.Projection.Mode.Name != ModeNeedsRebuild {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
|
|
result = core.ApplyEvent(RebuildCommitted{
|
|
ID: "vol-rebuild",
|
|
AchievedLSN: 64,
|
|
FlushedLSN: 64,
|
|
CheckpointLSN: 64,
|
|
})
|
|
if result.State.Recovery.Phase != RecoveryIdle {
|
|
t.Fatalf("recovery_phase=%s", result.State.Recovery.Phase)
|
|
}
|
|
if result.Projection.Recovery.Phase != RecoveryIdle {
|
|
t.Fatalf("projection_recovery_phase=%s", result.Projection.Recovery.Phase)
|
|
}
|
|
if result.Projection.Boundary.AchievedLSN != 64 {
|
|
t.Fatalf("achieved_lsn=%d", result.Projection.Boundary.AchievedLSN)
|
|
}
|
|
if result.Projection.Boundary.DurableLSN != 64 {
|
|
t.Fatalf("durable_lsn=%d", result.Projection.Boundary.DurableLSN)
|
|
}
|
|
if result.Projection.Boundary.CheckpointLSN != 64 {
|
|
t.Fatalf("checkpoint_lsn=%d", result.Projection.Boundary.CheckpointLSN)
|
|
}
|
|
if result.Projection.Mode.Name != ModeReplicaReady {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
if result.Projection.Publication.Healthy {
|
|
t.Fatal("replica rebuild completion must not overclaim healthy publication")
|
|
}
|
|
}
|
|
|
|
func TestPhase14_AssignmentChangeClearsStaleRecoveryTruth(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-reassign",
|
|
Epoch: 1,
|
|
Role: RoleReplica,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.30:9333", CtrlAddr: "10.0.0.30:9334", Version: 1}},
|
|
},
|
|
})
|
|
core.ApplyEvent(RoleApplied{ID: "vol-reassign"})
|
|
core.ApplyEvent(ReceiverReadyObserved{ID: "vol-reassign"})
|
|
core.ApplyEvent(CatchUpPlanned{ID: "vol-reassign", TargetLSN: 70})
|
|
core.ApplyEvent(RecoveryProgressObserved{ID: "vol-reassign", AchievedLSN: 41})
|
|
|
|
result := core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-reassign",
|
|
Epoch: 2,
|
|
Role: RoleReplica,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.31:9333", CtrlAddr: "10.0.0.31:9334", Version: 2}},
|
|
},
|
|
})
|
|
|
|
if result.State.Recovery.Phase != RecoveryIdle {
|
|
t.Fatalf("recovery_phase=%s", result.State.Recovery.Phase)
|
|
}
|
|
if result.Projection.Boundary.TargetLSN != 0 {
|
|
t.Fatalf("target_lsn=%d", result.Projection.Boundary.TargetLSN)
|
|
}
|
|
if result.Projection.Boundary.AchievedLSN != 0 {
|
|
t.Fatalf("achieved_lsn=%d", result.Projection.Boundary.AchievedLSN)
|
|
}
|
|
if result.Projection.Mode.Name != ModeBootstrapPending {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
if result.Projection.Publication.Reason != "awaiting_role_apply" {
|
|
t.Fatalf("publication_reason=%q", result.Projection.Publication.Reason)
|
|
}
|
|
}
|
|
|
|
func TestPhase14_MultiReplicaCatchUpAggregatesUntilAllComplete(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-multi-catchup",
|
|
Epoch: 1,
|
|
Role: RolePrimary,
|
|
RecoveryTarget: SessionCatchUp,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.40:9333", CtrlAddr: "10.0.0.40:9334", Version: 1}},
|
|
{ReplicaID: "replica-2", Endpoint: Endpoint{DataAddr: "10.0.0.41:9333", CtrlAddr: "10.0.0.41:9334", Version: 1}},
|
|
},
|
|
})
|
|
core.ApplyEvent(RoleApplied{ID: "vol-multi-catchup"})
|
|
core.ApplyEvent(ShipperConfiguredObserved{ID: "vol-multi-catchup"})
|
|
core.ApplyEvent(ShipperConnectedObserved{ID: "vol-multi-catchup"})
|
|
|
|
result := core.ApplyEvent(CatchUpPlanned{ID: "vol-multi-catchup", ReplicaID: "replica-1", TargetLSN: 100})
|
|
if result.State.Recovery.Phase != RecoveryCatchingUp {
|
|
t.Fatalf("phase_after_first_plan=%s", result.State.Recovery.Phase)
|
|
}
|
|
result = core.ApplyEvent(CatchUpPlanned{ID: "vol-multi-catchup", ReplicaID: "replica-2", TargetLSN: 100})
|
|
if result.State.Recovery.Phase != RecoveryCatchingUp {
|
|
t.Fatalf("phase_after_second_plan=%s", result.State.Recovery.Phase)
|
|
}
|
|
|
|
result = core.ApplyEvent(RecoveryProgressObserved{ID: "vol-multi-catchup", ReplicaID: "replica-1", AchievedLSN: 100})
|
|
if result.Projection.Recovery.AchievedLSN != 0 {
|
|
t.Fatalf("aggregate_achieved_lsn=%d", result.Projection.Recovery.AchievedLSN)
|
|
}
|
|
|
|
result = core.ApplyEvent(CatchUpCompleted{ID: "vol-multi-catchup", ReplicaID: "replica-1", AchievedLSN: 100})
|
|
if result.State.Recovery.Phase != RecoveryCatchingUp {
|
|
t.Fatalf("phase_after_first_complete=%s", result.State.Recovery.Phase)
|
|
}
|
|
if result.Projection.Boundary.DurableLSN != 0 {
|
|
t.Fatalf("durable_lsn_after_first_complete=%d", result.Projection.Boundary.DurableLSN)
|
|
}
|
|
if result.Projection.Publication.Healthy {
|
|
t.Fatal("publication should not become healthy before all replicas complete")
|
|
}
|
|
|
|
result = core.ApplyEvent(CatchUpCompleted{ID: "vol-multi-catchup", ReplicaID: "replica-2", AchievedLSN: 100})
|
|
if result.State.Recovery.Phase != RecoveryIdle {
|
|
t.Fatalf("phase_after_all_complete=%s", result.State.Recovery.Phase)
|
|
}
|
|
if result.Projection.Boundary.DurableLSN != 100 {
|
|
t.Fatalf("durable_lsn_after_all_complete=%d", result.Projection.Boundary.DurableLSN)
|
|
}
|
|
if result.Projection.Mode.Name != ModePublishHealthy {
|
|
t.Fatalf("mode_after_all_complete=%s", result.Projection.Mode.Name)
|
|
}
|
|
if !result.Projection.Publication.Healthy {
|
|
t.Fatal("publication should become healthy after all replicas complete")
|
|
}
|
|
}
|