From 10ce19cb17b6e62a9c9114131a145a4e9b81b15b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 19 Oct 2026 00:55:41 -0700 Subject: [PATCH] test: poll for TopologyId convergence in multi-master failover tests A restarted goraft master wipes its raft state and only learns the TopologyId after the leader re-admits it and replays the entry. That trails HTTP readiness by several seconds, so the one-shot check in assertTopologyIdConsistent raced the propagation (CI run 39199033687: TopologyId applied on the follower ~2s after the test gave up). --- test/multi_master/failover_test.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/test/multi_master/failover_test.go b/test/multi_master/failover_test.go index 9b491c89cc..acace3396e 100644 --- a/test/multi_master/failover_test.go +++ b/test/multi_master/failover_test.go @@ -290,9 +290,31 @@ func TestLeaderConsistencyAcrossNodes(t *testing.T) { assertTopologyIdConsistent(t, mc, topologyId) } -// assertTopologyIdConsistent verifies that all running nodes report the expected TopologyId. +// assertTopologyIdConsistent waits until all running nodes report the expected +// TopologyId. A restarted node only discovers it after the leader has admitted +// it back into raft — /dir/status is proxied to the leader, so a node that has +// not yet learned the leader reports its own empty id. That admission trails +// HTTP readiness by several seconds, so a one-shot check races the propagation. func assertTopologyIdConsistent(t *testing.T, mc *MasterCluster, expectedId string) { t.Helper() + deadline := time.Now().Add(waitTimeout) + for time.Now().Before(deadline) { + consistent := true + for i := range 3 { + if !mc.IsNodeRunning(i) { + continue + } + if id, err := mc.GetTopologyId(i); err != nil || id != expectedId { + consistent = false + break + } + } + if consistent { + return + } + time.Sleep(waitTick) + } + // Timed out: report what each node actually sees. for i := range 3 { if !mc.IsNodeRunning(i) { continue