test: poll for TopologyId convergence in multi-master failover tests

A restarted goraft master wipes its raft state and only learns the
TopologyId after the leader re-admits it and replays the entry. That
trails HTTP readiness by several seconds, so the one-shot check in
assertTopologyIdConsistent raced the propagation (CI run 39199033687:
TopologyId applied on the follower ~2s after the test gave up).
This commit is contained in:
Chris Lu committed 2026-10-19 00:55:41 -07:00
1 parent e20640acb4
commit 10ce19cb17
1 file changed
+23 -1
+23 -1
View File
@@ -290,9 +290,31 @@ func TestLeaderConsistencyAcrossNodes(t *testing.T) {
assertTopologyIdConsistent(t, mc, topologyId)
}
// assertTopologyIdConsistent verifies that all running nodes report the expected TopologyId.
// assertTopologyIdConsistent waits until all running nodes report the expected
// TopologyId. A restarted node only discovers it after the leader has admitted
// it back into raft — /dir/status is proxied to the leader, so a node that has
// not yet learned the leader reports its own empty id. That admission trails
// HTTP readiness by several seconds, so a one-shot check races the propagation.
func assertTopologyIdConsistent(t *testing.T, mc *MasterCluster, expectedId string) {
t.Helper()
deadline := time.Now().Add(waitTimeout)
for time.Now().Before(deadline) {
consistent := true
for i := range 3 {
if !mc.IsNodeRunning(i) {
continue
}
if id, err := mc.GetTopologyId(i); err != nil || id != expectedId {
consistent = false
break
}
}
if consistent {
return
}
time.Sleep(waitTick)
}
// Timed out: report what each node actually sees.
for i := range 3 {
if !mc.IsNodeRunning(i) {
continue