weed/server: verify data shards against bitrot sidecar before ec decode (#12422) (#12427)

* weed/server: verify data shards against bitrot sidecar before ec decode (#12422) * ec.decode: verify data shards against the mounted volume's bitrot protection The decode looked up only the generation-0 .ecsum beside the data base and each location's data directory, so it missed a sidecar kept with the index and ignored the generation the volume actually serves. Take the protection the volume's per-disk runtimes resolved instead (On, else Invalid, else Off), and test a corrupt block past the first. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * ec.decode: check for live entries before verifying the shards A volume whose needles are all deleted gets its shards purged on the no-live-entries status; a failing checksum must not keep it around. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * seaweed-volume: verify data shards against the bitrot protection before a decode Mirrors the Go volume server: the decode takes the protection the volume's per-disk runtimes resolved and refuses a data shard that fails its .ecsum checksum, after the no-live-entries check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: a decode refuses a data shard that fails its .ecsum checksum Runs against both volume server implementations: the refused decode keeps every shard and the sidecar, and the needle stays readable through EC. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * ec.decode: reload the bitrot sidecar before verifying the shards A rebuild's backfill writes the sidecar without reloading the mounted runtimes, which then still report no protection; resolve it again across the EC metadata directories, as a shard mount does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * seaweed-volume: reload the bitrot sidecar before verifying the shards Mirrors the Go volume server: the decode job resolves the sidecar again on every runtime of the volume before choosing its protection. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: a decode finds a sidecar written after the shards were mounted Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: read through the EC shards after the refused decode The source volume stayed mounted, so the final read never touched the shards it is meant to check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * ec.decode: refuse when a sidecar reload reports another layout ReloadBitrotSidecar logged and dropped the layout-mismatch error, so the runtime stayed Off and the decode skipped the data-shard check. It now returns that error and the decode refuses on it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * seaweed-volume: refuse a decode when a sidecar reload reports another layout Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: a decode refuses a late sidecar that records another layout Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * ec.decode: refuse when an unmount left no runtime to resolve the sidecar Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * seaweed-volume: refuse a decode when an unmount left no runtime to resolve the sidecar Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 12:07:51 +01:00 · 2026-11-05 10:11:09 -08:00
1 parent 80f32bc074
commit e44c16802d
@@ -399,6 +399,52 @@ func verifyShardFileBlocks(path string, entry *volume_server_pb.EcShardChecksums
 	return mismatched, nil
 }

+// SelectBitrotProtection picks the protection of a volume's per-disk runtimes:
+// the first On, else the first Invalid, else Off. After a restart only the
+// runtime whose disk holds the sidecar resolves it.
+func SelectBitrotProtection(volumes []*EcVolume) (*volume_server_pb.EcBitrotProtection, BitrotStatus) {
+	var invalid bool
+	for _, v := range volumes {
+		prot, status := v.BitrotProtection()
+		switch status {
+		case BitrotOn:
+			return prot, status
+		case BitrotInvalid:
+			invalid = true
+		}
+	}
+	if invalid {
+		return nil, BitrotInvalid
+	}
+	return nil, BitrotOff
+}
+
+// VerifyEcVolumeDataShards checks the data shards a decode reads against the
+// volume's bitrot protection: a corrupt shard copied into the .dat is lost for
+// good once the caller deletes the parity that could repair it.
+func VerifyEcVolumeDataShards(prot *volume_server_pb.EcBitrotProtection, status BitrotStatus, shardFileNames []string) error {
+	switch status {
+	case BitrotOff:
+		return nil
+	case BitrotInvalid:
+		return fmt.Errorf("bitrot sidecar is malformed or unverifiable")
+	}
+	for shardId, shardPath := range shardFileNames {
+		entry := shardChecksums(prot, uint32(shardId))
+		if entry == nil {
+			continue
+		}
+		mismatched, err := verifyShardFileBlocks(shardPath, entry, int64(prot.BlockSize))
+		if err != nil {
+			return fmt.Errorf("verify data shard %d %s: %w", shardId, shardPath, err)
+		}
+		if len(mismatched) > 0 {
+			return fmt.Errorf("data shard %d %s fails its bitrot checksum in %d blocks", shardId, shardPath, len(mismatched))
+		}
+	}
+	return nil
+}
+
 // ComputeProtectionFromShards builds a complete EcBitrotProtection by reading
 // the on-disk bytes of every shard for a generation — the trust-on-first-use
 // backfill primitive for EC volumes that were encoded before this feature. It
@@ -480,10 +526,13 @@ func (ev *EcVolume) BitrotProtection() (*volume_server_pb.EcBitrotProtection, Bi
 // already mounted — a shard delivery can bring the manifest with it, and the
 // receive path only writes the file, so without this the in-memory volume
 // keeps the protection state it resolved at mount (off) until a remount.
-func (ev *EcVolume) ReloadBitrotSidecar(additionalDirs ...string) {
-	if err := ev.loadActiveBitrotSidecar(additionalDirs...); err != nil {
+// It returns the layout-mismatch error a mount would fail on.
+func (ev *EcVolume) ReloadBitrotSidecar(additionalDirs ...string) error {
+	err := ev.loadActiveBitrotSidecar(additionalDirs...)
+	if err != nil {
 		glog.Warningf("ec volume %d: reload bitrot sidecar: %v", ev.VolumeId, err)
 	}
+	return err
 }

 // loadActiveBitrotSidecar loads the generation-0 checksum sidecar into the
@@ -346,6 +346,11 @@ func TestRemoveBitrotSidecars(t *testing.T) {
 }

 func corruptOneByte(t *testing.T, path string) {
+	t.Helper()
+	corruptOneByteAt(t, path, 0)
+}
+
+func corruptOneByteAt(t *testing.T, path string, offset int64) {
 	t.Helper()
 	f, err := os.OpenFile(path, os.O_RDWR, 0644)
 	if err != nil {
@@ -353,11 +358,70 @@ func corruptOneByte(t *testing.T, path string) {
 	}
 	defer f.Close()
 	var b [1]byte
-	if _, err := f.ReadAt(b[:], 0); err != nil {
+	if _, err := f.ReadAt(b[:], offset); err != nil {
 		t.Fatal(err)
 	}
 	b[0] ^= 0xff
-	if _, err := f.WriteAt(b[:], 0); err != nil {
+	if _, err := f.WriteAt(b[:], offset); err != nil {
 		t.Fatal(err)
 	}
 }
+
+func TestVerifyEcVolumeDataShards(t *testing.T) {
+	old := BitrotBlockSize
+	BitrotBlockSize = 1 << 20
+	t.Cleanup(func() { BitrotBlockSize = old })
+
+	dir := t.TempDir()
+	base := filepath.Join(dir, "42")
+	writeRandomDat(t, base, 25*1024*1024)
+	ctx := &ECContext{DataShards: 10, ParityShards: 4}
+	prot, err := WriteEcFiles(base, ctx)
+	if err != nil {
+		t.Fatalf("encode: %v", err)
+	}
+
+	shardFileNames := make([]string, ctx.DataShards)
+	for id := range shardFileNames {
+		shardFileNames[id] = base + ctx.ToExt(id)
+	}
+
+	if err := VerifyEcVolumeDataShards(prot, BitrotOn, shardFileNames); err != nil {
+		t.Fatalf("clean data shards: %v", err)
+	}
+	if err := VerifyEcVolumeDataShards(nil, BitrotInvalid, shardFileNames); err == nil {
+		t.Fatal("an invalid sidecar must fail the verification")
+	}
+
+	corruptOneByteAt(t, shardFileNames[3], 1<<20)
+	if err := VerifyEcVolumeDataShards(prot, BitrotOn, shardFileNames); err == nil {
+		t.Fatal("a corrupt second block of data shard 3 must fail the verification")
+	}
+	if err := VerifyEcVolumeDataShards(nil, BitrotOff, shardFileNames); err != nil {
+		t.Fatalf("an unprotected volume must pass: %v", err)
+	}
+}
+
+func TestSelectBitrotProtection(t *testing.T) {
+	prot := &volume_server_pb.EcBitrotProtection{BlockSize: 1}
+	runtime := func(p *volume_server_pb.EcBitrotProtection, status BitrotStatus) *EcVolume {
+		return &EcVolume{bitrot: p, bitrotStatus: status}
+	}
+	for _, tc := range []struct {
+		name     string
+		volumes  []*EcVolume
+		wantProt bool
+		want     BitrotStatus
+	}{
+		{"none", nil, false, BitrotOff},
+		{"all off", []*EcVolume{runtime(nil, BitrotOff), runtime(nil, BitrotOff)}, false, BitrotOff},
+		{"sidecar on the second disk", []*EcVolume{runtime(nil, BitrotOff), runtime(prot, BitrotOn)}, true, BitrotOn},
+		{"invalid beside off", []*EcVolume{runtime(nil, BitrotOff), runtime(nil, BitrotInvalid)}, false, BitrotInvalid},
+		{"on wins over invalid", []*EcVolume{runtime(nil, BitrotInvalid), runtime(prot, BitrotOn)}, true, BitrotOn},
+	} {
+		got, status := SelectBitrotProtection(tc.volumes)
+		if status != tc.want || (got != nil) != tc.wantProt {
+			t.Errorf("%s: got (%v, %v), want status %v", tc.name, got, status, tc.want)
+		}
+	}
+}
@@ -3,6 +3,7 @@ package erasure_coding
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"

 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
@@ -59,7 +60,7 @@ func TestNewEcVolumeTakesLayoutFromSidecarWhenVifHasNoConfig(t *testing.T) {
 // legacy layout.
 func TestNewEcVolumeFindsSidecarInTheIndexDirectory(t *testing.T) {
 	for _, tc := range []struct {
-		name    string
+		name     string
 		writeVif bool
 	}{
 		{name: "no vif at all"},
@@ -199,6 +200,16 @@ func TestReloadBitrotSidecarFindsItOnASiblingDisk(t *testing.T) {
 	if got := ev.bitrotStatus; got != BitrotOn {
 		t.Errorf("cross-disk reload: status = %v, want BitrotOn", got)
 	}
+
+	// A delivered sidecar that records another layout is reported, not
+	// swallowed: the decode refuses on it.
+	prot.EcShardConfig.ParityShards = 2
+	if err := SaveBitrotSidecar(BitrotSidecarPath(filepath.Join(sibling, "7"), 0), prot); err != nil {
+		t.Fatalf("seed mismatched .ecsum: %v", err)
+	}
+	if err := ev.ReloadBitrotSidecar(mine, sibling); err == nil || !strings.Contains(err.Error(), "refusing to serve") {
+		t.Errorf("mismatched-layout reload: err = %v, want the layout mismatch", err)
+	}
 }

 // "Does this volume already have a manifest" cannot be answered from one base