test(ec): cover 8-disk multi-disk rebuild and end-to-end decode (#9340)

Mirror the reporter's topology — single volume server with 8 hdd
locations (/mnt/d{1..8}/weed), .ecx and shards spread across disks the
way the volume server places them after a few rebuild rounds — and
exercise three states: clean, 0-byte residue on the rebuild disk,
0-byte residue on a sibling disk. Each variant must rebuild the missing
shards byte-identical to the originals AND decode the data shards back
through WriteDatFile to a .dat that matches the original byte-for-byte,
which is the real safety check that non-zero rebuilt files aren't
masking corrupted content.

weed/storage/erasure_coding/ec_rebuild_test.go

@@ -247,3 +247,199 @@ func equalUint32(a, b []uint32) bool {
 	}
 	return true
 }
+
+// TestRebuildEcFiles_EightDiskTopology reproduces the user's reported topology
+// from #9340: a single volume server with 8 hdd locations
+// (/mnt/d{1..8}/weed), .ecx on one disk, the surviving local shard on another,
+// freshly-copied shards on a third, shards 1 and 8 truly missing. The volume
+// server's VolumeEcShardsRebuild picks the .ecx-owning disk as rebuildLocation
+// and passes every other disk as additionalDirs; we mirror that contract here.
+//
+// The test exercises:
+//
+//  1. Clean state — must rebuild shards 1 and 8 byte-identical to the
+//     pre-removal originals.
+//  2. End-to-end decode — concatenating the data shards back through
+//     WriteDatFile reproduces the original .dat byte-for-byte, proving the
+//     rebuild is semantically correct (not just non-zero).
+//  3. Residue scenario — a previously aborted rebuild left 0-byte placeholders
+//     for shards 1 and 8 on the rebuildLocation. Pre-fix this caused
+//     findShardFile to return the ghost, the read loop to short-circuit on
+//     n==0, and the rebuild to silently no-op. Post-fix the ghosts are
+//     skipped, the rebuild succeeds, and the residue is removed.
+func TestRebuildEcFiles_EightDiskTopology(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		residueOn func(disks []string) [][2]string // (disk, ext) pairs to seed as 0-byte ghosts
+	}{
+		{name: "clean", residueOn: nil},
+		{
+			name: "residue_on_rebuild_disk",
+			residueOn: func(disks []string) [][2]string {
+				return [][2]string{
+					{disks[1], ToExt(1)}, // d2 == rebuildLocation in this fixture
+					{disks[1], ToExt(8)},
+				}
+			},
+		},
+		{
+			name: "residue_on_sibling_disk",
+			residueOn: func(disks []string) [][2]string {
+				return [][2]string{
+					{disks[3], ToExt(1)}, // a totally unrelated sibling disk
+					{disks[6], ToExt(8)},
+				}
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			runEightDiskRebuild(t, tc.residueOn)
+		})
+	}
+}
+
+func runEightDiskRebuild(t *testing.T, residueOn func(disks []string) [][2]string) {
+	t.Helper()
+
+	root := t.TempDir()
+	disks := make([]string, 8)
+	for i := range disks {
+		disks[i] = filepath.Join(root, "d"+string(rune('1'+i)), "weed")
+		mustMkdir(t, disks[i])
+	}
+
+	// Encode in a staging dir, then distribute shards across the 8 disks the
+	// way the volume server would after a couple of rebuild rounds:
+	// .ecx + .vif + a few shards on d2 (this is rebuildLocation), the
+	// existing shard 2 on d1, copied shards spread across d3..d8.
+	stagingBase := filepath.Join(root, "staging", "_109")
+	mustMkdir(t, filepath.Dir(stagingBase))
+	const datSize = int64(50 * 1024 * 1024)
+	originalShards, originalDat := encodeFixtureWithDat(t, stagingBase, datSize)
+
+	rebuildLocation := disks[1]                                                       // d2 — owns .ecx
+	homeDisk := []int{0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}                      // d1 holds shard 2 + most copied shards
+	homeDisk[3], homeDisk[4], homeDisk[5], homeDisk[6], homeDisk[7] = 2, 3, 4, 5, 6   // spread a few copied shards to d3..d7
+	homeDisk[10], homeDisk[11], homeDisk[12], homeDisk[13] = 7, 0, 1, 2               // and the rest across d8/d1/d2/d3
+	missing := map[int]struct{}{1: {}, 8: {}}
+
+	rebuildBase := filepath.Join(rebuildLocation, "collect_109")
+	mustRename(t, stagingBase+".vif", rebuildBase+".vif")
+	if _, err := os.Stat(stagingBase + ".ecx"); err == nil {
+		mustRename(t, stagingBase+".ecx", rebuildBase+".ecx")
+	}
+	for i := 0; i < TotalShardsCount; i++ {
+		if _, gone := missing[i]; gone {
+			mustRemove(t, stagingBase+ToExt(i))
+			continue
+		}
+		dst := filepath.Join(disks[homeDisk[i]], "collect_109"+ToExt(i))
+		mustRename(t, stagingBase+ToExt(i), dst)
+	}
+
+	if residueOn != nil {
+		for _, ent := range residueOn(disks) {
+			ghost := filepath.Join(ent[0], "collect_109"+ent[1])
+			mustWriteFile(t, ghost, nil)
+		}
+	}
+
+	// Mirror VolumeEcShardsRebuild's contract: rebuildLocation is the disk
+	// with .ecx; every other disk goes into additionalDirs.
+	var additionalDirs []string
+	for i, d := range disks {
+		if i == 1 {
+			continue
+		}
+		additionalDirs = append(additionalDirs, d)
+	}
+
+	rebuilt, err := RebuildEcFiles(rebuildBase, additionalDirs...)
+	if err != nil {
+		t.Fatalf("RebuildEcFiles: %v", err)
+	}
+	sort.Slice(rebuilt, func(i, j int) bool { return rebuilt[i] < rebuilt[j] })
+	if want := []uint32{1, 8}; !equalUint32(rebuilt, want) {
+		t.Fatalf("rebuilt = %v, want %v", rebuilt, want)
+	}
+
+	// Each rebuilt shard must match the original encoding bit-for-bit.
+	for idx := range missing {
+		got := mustReadFile(t, rebuildBase+ToExt(idx))
+		if !bytes.Equal(got, originalShards[idx]) {
+			t.Fatalf("rebuilt shard %d does not match the original encoding", idx)
+		}
+	}
+
+	// Any 0-byte residue we seeded on a sibling disk must be removed; residue
+	// at the rebuild location's output path is fine — it gets overwritten.
+	if residueOn != nil {
+		for _, ent := range residueOn(disks) {
+			ghost := filepath.Join(ent[0], "collect_109"+ent[1])
+			if ent[0] == rebuildLocation {
+				continue // overwritten by the output file
+			}
+			if _, statErr := os.Stat(ghost); !os.IsNotExist(statErr) {
+				t.Fatalf("residue %s was not removed (statErr=%v)", ghost, statErr)
+			}
+		}
+	}
+
+	// Decode all 10 data shards back through WriteDatFile and verify the
+	// reconstructed .dat matches the original byte-for-byte. This is the real
+	// safety check — non-zero rebuilt files aren't enough; they must contain
+	// the right data.
+	dataShardPaths := make([]string, DataShardsCount)
+	for i := 0; i < DataShardsCount; i++ {
+		if _, gone := missing[i]; gone {
+			dataShardPaths[i] = rebuildBase + ToExt(i)
+			continue
+		}
+		dataShardPaths[i] = filepath.Join(disks[homeDisk[i]], "collect_109"+ToExt(i))
+	}
+	decodedBase := filepath.Join(root, "decoded", "out")
+	mustMkdir(t, filepath.Dir(decodedBase))
+	if err := WriteDatFile(decodedBase, datSize, dataShardPaths); err != nil {
+		t.Fatalf("WriteDatFile: %v", err)
+	}
+	decoded := mustReadFile(t, decodedBase+".dat")
+	if !bytes.Equal(decoded, originalDat) {
+		t.Fatalf("decoded .dat does not match original (len got=%d want=%d)", len(decoded), len(originalDat))
+	}
+}
+
+// encodeFixtureWithDat is encodeFixture but also returns the original .dat
+// bytes so the caller can verify a full encode→rebuild→decode round trip.
+func encodeFixtureWithDat(t *testing.T, baseFileName string, datSize int64) (shards [TotalShardsCount][]byte, dat []byte) {
+	t.Helper()
+
+	dat = make([]byte, datSize)
+	if _, err := rand.Read(dat); err != nil {
+		t.Fatalf("rand: %v", err)
+	}
+	if err := os.WriteFile(baseFileName+".dat", dat, 0644); err != nil {
+		t.Fatalf("write dat: %v", err)
+	}
+	if err := WriteEcFiles(baseFileName); err != nil {
+		t.Fatalf("WriteEcFiles: %v", err)
+	}
+	if err := WriteSortedFileFromIdx(baseFileName, ".ecx"); err != nil {
+		// No .idx in this fixture (we don't run any reads), so this is best-effort.
+		t.Logf("WriteSortedFileFromIdx (best-effort): %v", err)
+	}
+	vif := &volume_server_pb.VolumeInfo{
+		Version:     3,
+		DatFileSize: datSize,
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards:   DataShardsCount,
+			ParityShards: ParityShardsCount,
+		},
+	}
+	if err := volume_info.SaveVolumeInfo(baseFileName+".vif", vif); err != nil {
+		t.Fatalf("save vif: %v", err)
+	}
+	for i := 0; i < TotalShardsCount; i++ {
+		shards[i] = mustReadFile(t, baseFileName+ToExt(i))
+	}
+	return
+}