From f4bad510c9dcfad528ecee9fc5deebdb68b9a300 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 13 Sep 2026 21:05:55 -0700 Subject: [PATCH] test/ec: pin rack in seedAndSpread volume.grow to stop silent no-ops (#11299) seedAndSpread() calls `volume.grow -dataNode X` without pinning the rack. The master's grow picks the rack by weighted-random when -rack is unset, and only one of the three racks holds the requested data node, so an unpinned grow lands on the wrong rack two times out of three. The VolumeGrow RPC swallows the "No matching data node" failure for non-cache collections, so those grows count as success without creating a volume. The per-server cap (maxGrowsPerServer=4) is then exhausted by silent no-ops before the volumes ever spread, and seedAndSpread times out with "volumes never spread across >=2 disks on all 3 nodes". Pin -dataCenter dc1 and -rack rack{i} alongside -dataNode so every grow reaches the target node. This removes the timing-sensitive assumption that made TestECVacuumDuplicateShardClaimAcrossDisks flaky. --- test/erasure_coding/chaos_lifecycle_test.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/erasure_coding/chaos_lifecycle_test.go b/test/erasure_coding/chaos_lifecycle_test.go index a74744323..27185f8e3 100644 --- a/test/erasure_coding/chaos_lifecycle_test.go +++ b/test/erasure_coding/chaos_lifecycle_test.go @@ -748,8 +748,18 @@ func (r *chaosRun) seedAndSpread() { for i := 0; i < chaosServerCount; i++ { server := "127.0.0.1:" + chaosVolumePort(i) if spread[server] < 2 && growsPerServer[server] < maxGrowsPerServer { + // Pin the data center and rack as well as the data node. The + // master's grow picks the rack by weighted random when -rack is + // unset, and only one of the three racks holds the requested + // data node, so an unpinned grow lands on the wrong rack two + // times out of three and the VolumeGrow RPC swallows the + // "No matching data node" error (non-cache grows ignore the + // internal failure). Those silent no-ops exhaust the per-server + // cap before the volumes ever spread, so seedAndSpread times + // out. Pinning the rack makes every grow reach the target node. out, gerr := captureCommandOutput(r.t, shell.Commands[findCommandIndex("volume.grow")], - []string{"-collection", chaosCollection, "-dataNode", server, "-count", "1"}, r.env) + []string{"-collection", chaosCollection, "-dataCenter", "dc1", + "-rack", fmt.Sprintf("rack%d", i), "-dataNode", server, "-count", "1"}, r.env) // Only count successful grows toward the cap: a transient // collectTopologyInfo or VolumeGrow RPC error would otherwise // exhaust the retry budget without creating any volume, and