From 1690bb2a03cd9b96619cd22a2d80039e74a6b536 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 8 May 2026 10:00:48 -0700 Subject: [PATCH] test(ec): cover 8-disk multi-disk rebuild and end-to-end decode (#9340) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror the reporter's topology — single volume server with 8 hdd locations (/mnt/d{1..8}/weed), .ecx and shards spread across disks the way the volume server places them after a few rebuild rounds — and exercise three states: clean, 0-byte residue on the rebuild disk, 0-byte residue on a sibling disk. Each variant must rebuild the missing shards byte-identical to the originals AND decode the data shards back through WriteDatFile to a .dat that matches the original byte-for-byte, which is the real safety check that non-zero rebuilt files aren't masking corrupted content. --- .../storage/erasure_coding/ec_rebuild_test.go | 196 ++++++++++++++++++ 1 file changed, 196 insertions(+) diff --git a/weed/storage/erasure_coding/ec_rebuild_test.go b/weed/storage/erasure_coding/ec_rebuild_test.go index 36b60bf4f..caac7ae7f 100644 --- a/weed/storage/erasure_coding/ec_rebuild_test.go +++ b/weed/storage/erasure_coding/ec_rebuild_test.go @@ -247,3 +247,199 @@ func equalUint32(a, b []uint32) bool { } return true } + +// TestRebuildEcFiles_EightDiskTopology reproduces the user's reported topology +// from #9340: a single volume server with 8 hdd locations +// (/mnt/d{1..8}/weed), .ecx on one disk, the surviving local shard on another, +// freshly-copied shards on a third, shards 1 and 8 truly missing. The volume +// server's VolumeEcShardsRebuild picks the .ecx-owning disk as rebuildLocation +// and passes every other disk as additionalDirs; we mirror that contract here. +// +// The test exercises: +// +// 1. Clean state — must rebuild shards 1 and 8 byte-identical to the +// pre-removal originals. +// 2. End-to-end decode — concatenating the data shards back through +// WriteDatFile reproduces the original .dat byte-for-byte, proving the +// rebuild is semantically correct (not just non-zero). +// 3. Residue scenario — a previously aborted rebuild left 0-byte placeholders +// for shards 1 and 8 on the rebuildLocation. Pre-fix this caused +// findShardFile to return the ghost, the read loop to short-circuit on +// n==0, and the rebuild to silently no-op. Post-fix the ghosts are +// skipped, the rebuild succeeds, and the residue is removed. +func TestRebuildEcFiles_EightDiskTopology(t *testing.T) { + for _, tc := range []struct { + name string + residueOn func(disks []string) [][2]string // (disk, ext) pairs to seed as 0-byte ghosts + }{ + {name: "clean", residueOn: nil}, + { + name: "residue_on_rebuild_disk", + residueOn: func(disks []string) [][2]string { + return [][2]string{ + {disks[1], ToExt(1)}, // d2 == rebuildLocation in this fixture + {disks[1], ToExt(8)}, + } + }, + }, + { + name: "residue_on_sibling_disk", + residueOn: func(disks []string) [][2]string { + return [][2]string{ + {disks[3], ToExt(1)}, // a totally unrelated sibling disk + {disks[6], ToExt(8)}, + } + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + runEightDiskRebuild(t, tc.residueOn) + }) + } +} + +func runEightDiskRebuild(t *testing.T, residueOn func(disks []string) [][2]string) { + t.Helper() + + root := t.TempDir() + disks := make([]string, 8) + for i := range disks { + disks[i] = filepath.Join(root, "d"+string(rune('1'+i)), "weed") + mustMkdir(t, disks[i]) + } + + // Encode in a staging dir, then distribute shards across the 8 disks the + // way the volume server would after a couple of rebuild rounds: + // .ecx + .vif + a few shards on d2 (this is rebuildLocation), the + // existing shard 2 on d1, copied shards spread across d3..d8. + stagingBase := filepath.Join(root, "staging", "_109") + mustMkdir(t, filepath.Dir(stagingBase)) + const datSize = int64(50 * 1024 * 1024) + originalShards, originalDat := encodeFixtureWithDat(t, stagingBase, datSize) + + rebuildLocation := disks[1] // d2 — owns .ecx + homeDisk := []int{0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} // d1 holds shard 2 + most copied shards + homeDisk[3], homeDisk[4], homeDisk[5], homeDisk[6], homeDisk[7] = 2, 3, 4, 5, 6 // spread a few copied shards to d3..d7 + homeDisk[10], homeDisk[11], homeDisk[12], homeDisk[13] = 7, 0, 1, 2 // and the rest across d8/d1/d2/d3 + missing := map[int]struct{}{1: {}, 8: {}} + + rebuildBase := filepath.Join(rebuildLocation, "collect_109") + mustRename(t, stagingBase+".vif", rebuildBase+".vif") + if _, err := os.Stat(stagingBase + ".ecx"); err == nil { + mustRename(t, stagingBase+".ecx", rebuildBase+".ecx") + } + for i := 0; i < TotalShardsCount; i++ { + if _, gone := missing[i]; gone { + mustRemove(t, stagingBase+ToExt(i)) + continue + } + dst := filepath.Join(disks[homeDisk[i]], "collect_109"+ToExt(i)) + mustRename(t, stagingBase+ToExt(i), dst) + } + + if residueOn != nil { + for _, ent := range residueOn(disks) { + ghost := filepath.Join(ent[0], "collect_109"+ent[1]) + mustWriteFile(t, ghost, nil) + } + } + + // Mirror VolumeEcShardsRebuild's contract: rebuildLocation is the disk + // with .ecx; every other disk goes into additionalDirs. + var additionalDirs []string + for i, d := range disks { + if i == 1 { + continue + } + additionalDirs = append(additionalDirs, d) + } + + rebuilt, err := RebuildEcFiles(rebuildBase, additionalDirs...) + if err != nil { + t.Fatalf("RebuildEcFiles: %v", err) + } + sort.Slice(rebuilt, func(i, j int) bool { return rebuilt[i] < rebuilt[j] }) + if want := []uint32{1, 8}; !equalUint32(rebuilt, want) { + t.Fatalf("rebuilt = %v, want %v", rebuilt, want) + } + + // Each rebuilt shard must match the original encoding bit-for-bit. + for idx := range missing { + got := mustReadFile(t, rebuildBase+ToExt(idx)) + if !bytes.Equal(got, originalShards[idx]) { + t.Fatalf("rebuilt shard %d does not match the original encoding", idx) + } + } + + // Any 0-byte residue we seeded on a sibling disk must be removed; residue + // at the rebuild location's output path is fine — it gets overwritten. + if residueOn != nil { + for _, ent := range residueOn(disks) { + ghost := filepath.Join(ent[0], "collect_109"+ent[1]) + if ent[0] == rebuildLocation { + continue // overwritten by the output file + } + if _, statErr := os.Stat(ghost); !os.IsNotExist(statErr) { + t.Fatalf("residue %s was not removed (statErr=%v)", ghost, statErr) + } + } + } + + // Decode all 10 data shards back through WriteDatFile and verify the + // reconstructed .dat matches the original byte-for-byte. This is the real + // safety check — non-zero rebuilt files aren't enough; they must contain + // the right data. + dataShardPaths := make([]string, DataShardsCount) + for i := 0; i < DataShardsCount; i++ { + if _, gone := missing[i]; gone { + dataShardPaths[i] = rebuildBase + ToExt(i) + continue + } + dataShardPaths[i] = filepath.Join(disks[homeDisk[i]], "collect_109"+ToExt(i)) + } + decodedBase := filepath.Join(root, "decoded", "out") + mustMkdir(t, filepath.Dir(decodedBase)) + if err := WriteDatFile(decodedBase, datSize, dataShardPaths); err != nil { + t.Fatalf("WriteDatFile: %v", err) + } + decoded := mustReadFile(t, decodedBase+".dat") + if !bytes.Equal(decoded, originalDat) { + t.Fatalf("decoded .dat does not match original (len got=%d want=%d)", len(decoded), len(originalDat)) + } +} + +// encodeFixtureWithDat is encodeFixture but also returns the original .dat +// bytes so the caller can verify a full encode→rebuild→decode round trip. +func encodeFixtureWithDat(t *testing.T, baseFileName string, datSize int64) (shards [TotalShardsCount][]byte, dat []byte) { + t.Helper() + + dat = make([]byte, datSize) + if _, err := rand.Read(dat); err != nil { + t.Fatalf("rand: %v", err) + } + if err := os.WriteFile(baseFileName+".dat", dat, 0644); err != nil { + t.Fatalf("write dat: %v", err) + } + if err := WriteEcFiles(baseFileName); err != nil { + t.Fatalf("WriteEcFiles: %v", err) + } + if err := WriteSortedFileFromIdx(baseFileName, ".ecx"); err != nil { + // No .idx in this fixture (we don't run any reads), so this is best-effort. + t.Logf("WriteSortedFileFromIdx (best-effort): %v", err) + } + vif := &volume_server_pb.VolumeInfo{ + Version: 3, + DatFileSize: datSize, + EcShardConfig: &volume_server_pb.EcShardConfig{ + DataShards: DataShardsCount, + ParityShards: ParityShardsCount, + }, + } + if err := volume_info.SaveVolumeInfo(baseFileName+".vif", vif); err != nil { + t.Fatalf("save vif: %v", err) + } + for i := 0; i < TotalShardsCount; i++ { + shards[i] = mustReadFile(t, baseFileName+ToExt(i)) + } + return +}