package multi_master import ( "fmt" "math/rand/v2" "os" "strconv" "testing" "time" ) // chaosRounds is one stop or start per round, enough to walk in and out of // quorum several times without turning this into a soak test. const chaosRounds = 12 // TestRandomStartStopElection bounces masters at random and holds the election // to the two things it must never get wrong: two masters claiming leadership at // once, and a quorum that comes back without agreeing on one. The cluster's // identity has to survive the whole walk — a master that re-mints a TopologyId // here is the split brain SetTopologyId kills its peers over. func TestRandomStartStopElection(t *testing.T) { for _, impl := range raftImplementations { t.Run(impl.name, func(t *testing.T) { seed := chaosSeed(t) t.Logf("seed %d — replay this walk with MULTI_MASTER_IT_SEED=%d", seed, seed) rng := rand.New(rand.NewPCG(seed, seed)) mc := NewMasterCluster(t, impl.raftHashicorp) for i := range 3 { mc.StartNode(i) } if err := mc.WaitForLeader(waitTimeout); err != nil { mc.DumpLogs() t.Fatalf("cluster did not elect a leader: %v", err) } topologyId, err := mc.WaitForTopologyId(waitTimeout) if err != nil { mc.DumpLogs() t.Fatalf("no initial TopologyId: %v", err) } for round := 1; round <= chaosRounds; round++ { target := rng.IntN(3) if mc.IsNodeRunning(target) { mc.StopNode(target) t.Logf("round %d: stopped master %d, %d left running", round, target, runningMasters(mc)) } else { mc.StartNode(target) t.Logf("round %d: started master %d, %d now running", round, target, runningMasters(mc)) } if err := waitForSettledElection(mc, leaderElectionTimeout); err != nil { mc.DumpLogs() t.Fatalf("round %d (seed %d): %v", round, seed, err) } // /dir/status proxies to the leader, so this reads the value the // cluster as a whole is carrying, not any one master's copy. if runningMasters(mc) >= 2 { id, err := mc.WaitForTopologyId(leaderElectionTimeout) if err != nil { mc.DumpLogs() t.Fatalf("round %d (seed %d): %v", round, seed, err) } if id != topologyId { mc.DumpLogs() t.Fatalf("round %d (seed %d): TopologyId changed from %s to %s", round, seed, topologyId, id) } } } // Everything back up, so the walk ends on a full cluster. for i := range 3 { mc.StartNode(i) } if _, err := waitForCommonLeader(mc, leaderElectionTimeout); err != nil { mc.DumpLogs() t.Fatalf("cluster did not recover after the walk (seed %d): %v", seed, err) } for i := range 3 { if err := waitForPeerCount(mc, i, 2, leaderElectionTimeout); err != nil { mc.DumpLogs() t.Fatalf("master %d does not see the full cluster (seed %d): %v", i, seed, err) } } id, err := mc.WaitForTopologyId(leaderElectionTimeout) if err != nil { mc.DumpLogs() t.Fatalf("no TopologyId after the walk (seed %d): %v", seed, err) } if id != topologyId { mc.DumpLogs() t.Fatalf("TopologyId changed from %s to %s over the walk (seed %d)", topologyId, id, seed) } }) } } // waitForSettledElection waits for a quorum to agree on one leader, and fails // if two masters claim leadership across consecutive polls — anything shorter // than that is a master on its way down. // // Below a quorum there is nothing to wait for: the walk moves on. A master that // has lost its quorum can keep claiming leadership for tens of seconds under // goraft, and it cannot commit anything in that window, so stepping down is a // liveness question rather than a safety one. TestTwoMastersDownAndRestart // holds that direction to account. func waitForSettledElection(mc *MasterCluster, timeout time.Duration) error { if runningMasters(mc) < 2 { return nil } var lastErr error splitPolls := 0 deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { claims, err := leaderClaims(mc) if err != nil { lastErr = err } if len(claims) > 1 { splitPolls++ if splitPolls > 1 { return fmt.Errorf("masters %v all claim leadership", claims) } time.Sleep(waitTick) continue } splitPolls = 0 if _, err := commonLeader(mc); err == nil { return nil } else { lastErr = err } time.Sleep(waitTick) } if lastErr == nil { lastErr = fmt.Errorf("cluster did not settle within %v", timeout) } return lastErr } // leaderClaims returns the running masters that call themselves leader. The // error reports masters that did not answer at all, which is a reason to keep // waiting rather than a verdict. func leaderClaims(mc *MasterCluster) (claims []int, err error) { for i := range 3 { if !mc.IsNodeRunning(i) { continue } cs, statusErr := mc.GetClusterStatus(i) if statusErr != nil { err = fmt.Errorf("master %d: %w", i, statusErr) continue } if cs.IsLeader { claims = append(claims, i) } } return claims, err } func runningMasters(mc *MasterCluster) int { count := 0 for i := range 3 { if mc.IsNodeRunning(i) { count++ } } return count } // chaosSeed picks the walk. It is random by default and always logged, so a // failure names the seed that reproduces it. func chaosSeed(t *testing.T) uint64 { t.Helper() if v := os.Getenv("MULTI_MASTER_IT_SEED"); v != "" { seed, err := strconv.ParseUint(v, 10, 64) if err != nil { t.Fatalf("MULTI_MASTER_IT_SEED %q: %v", v, err) } return seed } return uint64(time.Now().UnixNano()) }