weed/server: verify data shards against bitrot sidecar before ec decode (#12422) (#12427)

* weed/server: verify data shards against bitrot sidecar before ec decode (#12422)

* ec.decode: verify data shards against the mounted volume's bitrot protection

The decode looked up only the generation-0 .ecsum beside the data base and
each location's data directory, so it missed a sidecar kept with the index
and ignored the generation the volume actually serves.

Take the protection the volume's per-disk runtimes resolved instead (On, else
Invalid, else Off), and test a corrupt block past the first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* ec.decode: check for live entries before verifying the shards

A volume whose needles are all deleted gets its shards purged on the
no-live-entries status; a failing checksum must not keep it around.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* seaweed-volume: verify data shards against the bitrot protection before a decode

Mirrors the Go volume server: the decode takes the protection the volume's
per-disk runtimes resolved and refuses a data shard that fails its .ecsum
checksum, after the no-live-entries check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: a decode refuses a data shard that fails its .ecsum checksum

Runs against both volume server implementations: the refused decode keeps
every shard and the sidecar, and the needle stays readable through EC.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* ec.decode: reload the bitrot sidecar before verifying the shards

A rebuild's backfill writes the sidecar without reloading the mounted
runtimes, which then still report no protection; resolve it again across
the EC metadata directories, as a shard mount does.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* seaweed-volume: reload the bitrot sidecar before verifying the shards

Mirrors the Go volume server: the decode job resolves the sidecar again on
every runtime of the volume before choosing its protection.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: a decode finds a sidecar written after the shards were mounted

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: read through the EC shards after the refused decode

The source volume stayed mounted, so the final read never touched the
shards it is meant to check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* ec.decode: refuse when a sidecar reload reports another layout

ReloadBitrotSidecar logged and dropped the layout-mismatch error, so the
runtime stayed Off and the decode skipped the data-shard check. It now
returns that error and the decode refuses on it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* seaweed-volume: refuse a decode when a sidecar reload reports another layout

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: a decode refuses a late sidecar that records another layout

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* ec.decode: refuse when an unmount left no runtime to resolve the sidecar

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* seaweed-volume: refuse a decode when an unmount left no runtime to resolve the sidecar

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

seaweed-volume/src/server/grpc_server.rs

@@ -4353,7 +4353,7 @@ impl VolumeServer for VolumeGrpcService {
         // however often it reloaded. One delivered copy, reachable from all.
         let ec_metadata_dirs = store.ec_metadata_dirs();
         for ec_vol in store.find_all_ec_volumes_mut(vid) {
-            ec_vol.reload_bitrot_sidecar(&ec_metadata_dirs);
+            let _ = ec_vol.reload_bitrot_sidecar(&ec_metadata_dirs);
         }
         drop(store);
         self.state.volume_state_notify.notify_one();
@@ -7766,6 +7766,45 @@ impl EcDecodeJob {
             )));
         }
 
+        let shard_paths: Vec<String> = shard_dirs
+            .iter()
+            .enumerate()
+            .map(|(i, dir)| {
+                crate::storage::erasure_coding::ec_shard::EcVolumeShard::new(
+                    dir,
+                    &collection,
+                    vid,
+                    i as ShardId,
+                )
+                .file_name()
+            })
+            .collect();
+        // A rebuild's backfill writes the sidecar without reloading the
+        // mounted runtimes, so resolve it again the way a shard mount does.
+        // Any runtime left finds it on every disk; with none left (an unmount
+        // raced the decode) the protection would read as Off.
+        let (bitrot, bitrot_status) = {
+            let mut store = state.store.write().unwrap();
+            let ec_metadata_dirs = store.ec_metadata_dirs();
+            let runtimes = store.find_all_ec_volumes_mut(vid);
+            if runtimes.is_empty() {
+                return Err(Status::internal(format!(
+                    "ec volume {} was unmounted during the decode",
+                    vid
+                )));
+            }
+            for ec_vol in runtimes {
+                ec_vol
+                    .reload_bitrot_sidecar(&ec_metadata_dirs)
+                    .map_err(|e| Status::internal(format!("ec volume {}: {}", vid, e)))?;
+            }
+            crate::storage::erasure_coding::ec_volume::EcVolume::select_bitrot_protection(
+                &store.find_all_ec_volumes(vid),
+            )
+        };
+        ec_bitrot::verify_ec_volume_data_shards(bitrot.as_ref(), bitrot_status, &shard_paths)
+            .map_err(|e| Status::internal(format!("ec volume {}: {}", vid, e)))?;
+
         // Calculate .dat file size from .ecx entries (.ec00 lives on
         // its own disk, .ecx on the index disk).
         let dat_file_size = ec_decoder::find_dat_file_size_with_dirs(
@@ -14805,6 +14844,149 @@ mod tests {
         assert!(std::path::Path::new(&idx_sidecar).exists());
     }
 
+    /// A data shard that fails its .ecsum checksum must fail the decode and
+    /// leave every shard and the sidecar in place for a repair.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_volume_ec_shards_to_volume_rejects_a_corrupt_data_shard() {
+        let (service, _tmp, data, _idx, _) = make_split_idx_ec_decode_service(&[], false).await;
+        assert_corrupt_data_shard_rejected(&service, &data).await;
+    }
+
+    /// A rebuild's backfill writes the sidecar after the shards are mounted,
+    /// without reloading the runtimes: the decode must still find it.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_volume_ec_shards_to_volume_rejects_a_corrupt_shard_with_a_late_sidecar() {
+        let (service, _tmp, data, _idx, _) = make_split_idx_ec_decode_service(&[], false).await;
+        let sidecar = format!("{data}/1.ecsum");
+        let hidden = format!("{data}/1.ecsum.hidden");
+        std::fs::rename(&sidecar, &hidden).unwrap();
+        {
+            let mut store = service.state.store.write().unwrap();
+            for ec_vol in store.find_all_ec_volumes_mut(VolumeId(1)) {
+                ec_vol.reload_bitrot_sidecar(&[]).unwrap();
+                assert_eq!(
+                    ec_vol.bitrot_protection().1,
+                    crate::storage::erasure_coding::ec_bitrot::BitrotStatus::Off,
+                    "precondition: the mounted runtime holds no protection"
+                );
+            }
+        }
+        std::fs::rename(&hidden, &sidecar).unwrap();
+        assert_corrupt_data_shard_rejected(&service, &data).await;
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_volume_ec_shards_to_volume_refuses_a_late_sidecar_with_another_layout() {
+        use crate::storage::erasure_coding::ec_bitrot;
+        let (service, _tmp, data, _idx, _) = make_split_idx_ec_decode_service(&[], false).await;
+        let sidecar = format!("{data}/1.ecsum");
+        let hidden = format!("{data}/1.ecsum.hidden");
+        std::fs::rename(&sidecar, &hidden).unwrap();
+        {
+            let mut store = service.state.store.write().unwrap();
+            for ec_vol in store.find_all_ec_volumes_mut(VolumeId(1)) {
+                ec_vol.reload_bitrot_sidecar(&[]).unwrap();
+            }
+        }
+        let mut prot = ec_bitrot::load_bitrot_sidecar(&hidden).unwrap();
+        prot.ec_shard_config.as_mut().unwrap().parity_shards = 2;
+        ec_bitrot::save_bitrot_sidecar(&sidecar, &prot).unwrap();
+        std::fs::remove_file(&hidden).unwrap();
+
+        let err = service
+            .volume_ec_shards_to_volume(ec_shards_to_volume_request())
+            .await
+            .unwrap_err();
+
+        assert!(
+            err.message().contains("refusing to serve"),
+            "unexpected error: {}",
+            err.message()
+        );
+        assert!(!std::path::Path::new(&format!("{data}/1.dat")).exists());
+        for shard_id in 0..14 {
+            assert!(std::path::Path::new(&format!("{data}/1.ec{shard_id:02}")).exists());
+        }
+    }
+
+    /// An unmount between the job snapshot and the verification leaves no
+    /// runtime to resolve the sidecar: the job must refuse, not verify as Off.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_ec_decode_job_refuses_when_every_runtime_was_unmounted() {
+        let (service, _tmp, data, _idx, _) = make_split_idx_ec_decode_service(&[], false).await;
+        let job = {
+            let store = service.state.store.read().unwrap();
+            let (ec_vol, shard_dirs) = store
+                .collect_ec_shard_dirs(
+                    VolumeId(1),
+                    crate::storage::erasure_coding::ec_shard::MAX_SHARD_COUNT,
+                )
+                .unwrap();
+            let data_shards = ec_vol.data_shards as usize;
+            EcDecodeJob {
+                vid: VolumeId(1),
+                dat_dir: ec_vol.dir.clone(),
+                ecx_dir: ec_vol.ecx_actual_dir().to_string(),
+                idx_dir: ec_vol.dir_idx.clone(),
+                collection: ec_vol.collection.clone(),
+                vif_dat_file_size: ec_vol.dat_file_size,
+                large_block_size: ec_vol.large_block_size() as usize,
+                small_block_size: ec_vol.small_block_size() as usize,
+                shard_dirs: shard_dirs[..data_shards]
+                    .iter()
+                    .map(|d| d.clone().unwrap())
+                    .collect(),
+                needle_map_kind: store.needle_map_kind,
+            }
+        };
+        service
+            .volume_ec_shards_unmount(Request::new(
+                volume_server_pb::VolumeEcShardsUnmountRequest {
+                    volume_id: 1,
+                    shard_ids: (0..14).collect(),
+                    encode_ts_ns: 0,
+                },
+            ))
+            .await
+            .unwrap();
+        let shard = format!("{data}/1.ec03");
+        let mut bytes = std::fs::read(&shard).unwrap();
+        bytes[0] ^= 0xff;
+        std::fs::write(&shard, bytes).unwrap();
+
+        let err = job.run(&service.state).unwrap_err();
+
+        assert!(
+            err.message().contains("unmounted during the decode"),
+            "unexpected error: {}",
+            err.message()
+        );
+        assert!(!std::path::Path::new(&format!("{data}/1.dat")).exists());
+    }
+
+    async fn assert_corrupt_data_shard_rejected(service: &VolumeGrpcService, data: &str) {
+        let shard = format!("{data}/1.ec03");
+        let mut bytes = std::fs::read(&shard).unwrap();
+        bytes[0] ^= 0xff;
+        std::fs::write(&shard, bytes).unwrap();
+
+        let err = service
+            .volume_ec_shards_to_volume(ec_shards_to_volume_request())
+            .await
+            .unwrap_err();
+
+        assert!(
+            err.message().contains("data shard 3"),
+            "unexpected error: {}",
+            err.message()
+        );
+        assert!(!std::path::Path::new(&format!("{data}/1.dat")).exists());
+        assert!(std::path::Path::new(&format!("{data}/1.ecsum")).exists());
+        for shard_id in 0..14 {
+            assert!(std::path::Path::new(&format!("{data}/1.ec{shard_id:02}")).exists());
+        }
+    }
+
     /// Replaces the idx-dir .ecj with a FIFO: the decode's open of it blocks
     /// until [`release_ecj_fifo`] opens the write end.
     #[cfg(unix)]

seaweed-volume/src/storage/erasure_coding/ec_bitrot.rs

@@ -581,6 +581,51 @@ pub fn verify_shard_blocks(
     Ok(mismatched)
 }
 
+/// Checks the data shards a decode reads against the volume's bitrot
+/// protection: a corrupt shard copied into the .dat is lost for good once the
+/// caller deletes the parity that could repair it. Mirrors Go's
+/// `VerifyEcVolumeDataShards`.
+pub fn verify_ec_volume_data_shards(
+    prot: Option<&EcBitrotProtection>,
+    status: BitrotStatus,
+    shard_paths: &[String],
+) -> io::Result<()> {
+    let prot = match (status, prot) {
+        (BitrotStatus::Off, _) => return Ok(()),
+        (BitrotStatus::On, Some(prot)) => prot,
+        _ => {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                "bitrot sidecar is malformed or unverifiable",
+            ));
+        }
+    };
+    for (shard_id, path) in shard_paths.iter().enumerate() {
+        let Some(entry) = shard_checksums(prot, shard_id as u32) else {
+            continue;
+        };
+        let mismatched =
+            verify_shard_file_blocks(path, entry, prot.block_size as i64).map_err(|e| {
+                io::Error::new(
+                    e.kind(),
+                    format!("verify data shard {} {}: {}", shard_id, path, e),
+                )
+            })?;
+        if !mismatched.is_empty() {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!(
+                    "data shard {} {} fails its bitrot checksum in {} blocks",
+                    shard_id,
+                    path,
+                    mismatched.len()
+                ),
+            ));
+        }
+    }
+    Ok(())
+}
+
 /// Builds the `EcShardConfig` proto for the given layout. The bitrot sidecar
 /// carries its own top-level encode_uuid, so the nested config leaves it empty.
 pub fn ec_shard_config(data_shards: u32, parity_shards: u32, block_size: i64) -> EcShardConfig {

seaweed-volume/src/storage/erasure_coding/ec_volume.rs

@@ -238,8 +238,8 @@ pub struct EcVolume {
     /// no encode identity, so provenance is the only signal that a loaded
     /// manifest may describe a different encode run. Same purpose as
     /// `ecx_actual_dir`. Empty when no sidecar was found. Always committed in
-    /// the same step as `bitrot`/`bitrot_status`, so a reload whose `Err` gets
-    /// swallowed by `reload_bitrot_sidecar` cannot leave this describing a
+    /// the same step as `bitrot`/`bitrot_status`, so a reload whose `Err` a
+    /// caller ignores cannot leave this describing a
     /// sidecar other than the one those two fields actually hold.
     pub(crate) bitrot_source_dir: String,
 
@@ -802,14 +802,17 @@ impl EcVolume {
     /// shard delivery can bring the manifest with it, and the receive path only
     /// writes the file, so without this the in-memory volume keeps the
     /// protection state it resolved at mount (off) until a remount.
-    pub fn reload_bitrot_sidecar(&mut self, additional_dirs: &[String]) {
-        if let Err(e) = self.load_active_bitrot_sidecar(additional_dirs) {
+    /// Returns the layout-mismatch error a mount would fail on.
+    pub fn reload_bitrot_sidecar(&mut self, additional_dirs: &[String]) -> io::Result<()> {
+        let result = self.load_active_bitrot_sidecar(additional_dirs);
+        if let Err(e) = &result {
             tracing::warn!(
                 volume_id = self.volume_id.0,
                 error = %e,
                 "reload bitrot sidecar",
             );
         }
+        result
     }
 
     /// Load the generation-0 checksum sidecar into `self.bitrot`/`self.bitrot_status`.
@@ -888,8 +891,8 @@ impl EcVolume {
             self.parity_shards as usize,
         );
         // Provenance is committed together with the sidecar it describes, so a
-        // swallowed reload (`reload_bitrot_sidecar` logs and discards the `Err`
-        // above) cannot leave them disagreeing: the geometry-mismatch return
+        // reload whose `Err` above a caller ignores cannot leave them
+        // disagreeing: the geometry-mismatch return
         // happens before this point, so `bitrot`/`bitrot_status` below and
         // `bitrot_source_dir` here always describe the same load attempt.
         // Only a path that actually exists is a real source: `path` falls back
@@ -934,6 +937,31 @@ impl EcVolume {
         (self.bitrot.clone(), self.bitrot_status)
     }
 
+    /// The protection of a volume's per-disk runtimes: the first `On`, else
+    /// `Invalid`, else `Off`. After a restart only the runtime whose disk holds
+    /// the sidecar resolves it. Mirrors Go's `SelectBitrotProtection`.
+    pub(crate) fn select_bitrot_protection(
+        volumes: &[&EcVolume],
+    ) -> (
+        Option<crate::pb::volume_server_pb::EcBitrotProtection>,
+        crate::storage::erasure_coding::ec_bitrot::BitrotStatus,
+    ) {
+        use crate::storage::erasure_coding::ec_bitrot::BitrotStatus;
+        let mut invalid = false;
+        for v in volumes {
+            match v.bitrot_protection() {
+                (prot, BitrotStatus::On) => return (prot, BitrotStatus::On),
+                (_, BitrotStatus::Invalid) => invalid = true,
+                _ => {}
+            }
+        }
+        if invalid {
+            (None, BitrotStatus::Invalid)
+        } else {
+            (None, BitrotStatus::Off)
+        }
+    }
+
     /// Read-only EC bitrot checksum scrub of the LOCAL shards of this volume's
     /// active generation.
     ///

test/volume_server/grpc/ec_decode_bitrot_test.go

@@ -0,0 +1,125 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+func TestEcShardsToVolumeRejectsCorruptDataShard(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+	t.Run("sidecar loaded at mount", func(t *testing.T) {
+		testEcShardsToVolumeRejectsCorruptDataShard(t, false, 0, "data shard 3")
+	})
+	// A rebuild's backfill writes the sidecar after the shards are mounted.
+	t.Run("sidecar written after mount", func(t *testing.T) {
+		testEcShardsToVolumeRejectsCorruptDataShard(t, true, 0, "data shard 3")
+	})
+	t.Run("sidecar with another layout written after mount", func(t *testing.T) {
+		testEcShardsToVolumeRejectsCorruptDataShard(t, true, 2, "refusing to serve")
+	})
+}
+
+func testEcShardsToVolumeRejectsCorruptDataShard(t *testing.T, sidecarAfterMount bool, lateParityShards uint32, wantErr string) {
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(123)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 990007, 0x66771123)
+	payload := []byte("ec-shards-to-volume-corrupt-data-shard")
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId: volumeID,
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsGenerate: %v", err)
+	}
+	// Reads must go through the EC shards, not the source volume.
+	if _, err := grpcClient.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{
+		VolumeId: volumeID,
+	}); err != nil {
+		t.Fatalf("VolumeUnmount: %v", err)
+	}
+	dataDir := filepath.Join(clusterHarness.BaseDir(), "volume")
+	base := filepath.Join(dataDir, fmt.Sprintf("%d", volumeID))
+	if _, err := os.Stat(base + ".ecsum"); err != nil {
+		t.Fatalf("precondition: the encode wrote a bitrot sidecar: %v", err)
+	}
+	if sidecarAfterMount {
+		if err := os.Rename(base+".ecsum", base+".ecsum.hidden"); err != nil {
+			t.Fatalf("hide sidecar: %v", err)
+		}
+	}
+	if _, err := grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId: volumeID,
+		ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13},
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsMount: %v", err)
+	}
+	if sidecarAfterMount {
+		if err := os.Rename(base+".ecsum.hidden", base+".ecsum"); err != nil {
+			t.Fatalf("restore sidecar: %v", err)
+		}
+	}
+	if lateParityShards != 0 {
+		prot, err := erasure_coding.LoadBitrotSidecar(base + ".ecsum")
+		if err != nil {
+			t.Fatalf("load sidecar: %v", err)
+		}
+		prot.EcShardConfig.ParityShards = lateParityShards
+		if err := erasure_coding.SaveBitrotSidecar(base+".ecsum", prot); err != nil {
+			t.Fatalf("rewrite sidecar: %v", err)
+		}
+	}
+	shard := base + ".ec03"
+	content, err := os.ReadFile(shard)
+	if err != nil {
+		t.Fatalf("read %s: %v", shard, err)
+	}
+	content[0] ^= 0xff
+	if err := os.WriteFile(shard, content, 0644); err != nil {
+		t.Fatalf("corrupt %s: %v", shard, err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId: volumeID,
+	})
+	if err == nil || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("VolumeEcShardsToVolume with a corrupt data shard: %v", err)
+	}
+
+	for _, ext := range []string{".ecsum", ".ec00", ".ec03", ".ec13"} {
+		if _, statErr := os.Stat(base + ext); statErr != nil {
+			t.Fatalf("%s must survive the refused decode: %v", ext, statErr)
+		}
+	}
+
+	readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid)
+	readBody := framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusOK || string(readBody) != string(payload) {
+		t.Fatalf("EC read after the refused decode: status %d body %q", readResp.StatusCode, readBody)
+	}
+}

weed/server/volume_grpc_erasure_coding.go

@@ -1302,6 +1302,25 @@ func (vs *VolumeServer) VolumeEcShardsToVolume(ctx context.Context, req *volume_
 		return nil, status.Errorf(codes.FailedPrecondition, "ec volume %d %s", req.VolumeId, erasure_coding.EcNoLiveEntriesSubstring)
 	}
 
+	// A rebuild's backfill writes the sidecar without reloading the mounted
+	// runtimes, so resolve it again the way a shard mount does. Any runtime
+	// left finds it on every disk; with none left (an unmount raced the
+	// decode) the protection would read as Off.
+	ecVolumes := vs.store.FindAllEcVolumes(needle.VolumeId(req.VolumeId))
+	if len(ecVolumes) == 0 {
+		return nil, fmt.Errorf("ec volume %d was unmounted during the decode", req.VolumeId)
+	}
+	ecMetadataDirs := vs.store.EcMetadataDirs()
+	for _, ecVolume := range ecVolumes {
+		if err := ecVolume.ReloadBitrotSidecar(ecMetadataDirs...); err != nil {
+			return nil, fmt.Errorf("ec volume %d: %w", req.VolumeId, err)
+		}
+	}
+	prot, bitrotStatus := erasure_coding.SelectBitrotProtection(ecVolumes)
+	if err := erasure_coding.VerifyEcVolumeDataShards(prot, bitrotStatus, shardFileNames); err != nil {
+		return nil, fmt.Errorf("ec volume %d: %w", req.VolumeId, err)
+	}
+
 	// calculate .dat file size. Pass shard 0's resolved path: on a multi-disk
 	// server the volume's shards can sit on several disks, so the .ec00 is not
 	// necessarily beside the EcVolume's own base path.

weed/storage/erasure_coding/ec_bitrot.go

@@ -399,6 +399,52 @@ func verifyShardFileBlocks(path string, entry *volume_server_pb.EcShardChecksums
 	return mismatched, nil
 }
 
+// SelectBitrotProtection picks the protection of a volume's per-disk runtimes:
+// the first On, else the first Invalid, else Off. After a restart only the
+// runtime whose disk holds the sidecar resolves it.
+func SelectBitrotProtection(volumes []*EcVolume) (*volume_server_pb.EcBitrotProtection, BitrotStatus) {
+	var invalid bool
+	for _, v := range volumes {
+		prot, status := v.BitrotProtection()
+		switch status {
+		case BitrotOn:
+			return prot, status
+		case BitrotInvalid:
+			invalid = true
+		}
+	}
+	if invalid {
+		return nil, BitrotInvalid
+	}
+	return nil, BitrotOff
+}
+
+// VerifyEcVolumeDataShards checks the data shards a decode reads against the
+// volume's bitrot protection: a corrupt shard copied into the .dat is lost for
+// good once the caller deletes the parity that could repair it.
+func VerifyEcVolumeDataShards(prot *volume_server_pb.EcBitrotProtection, status BitrotStatus, shardFileNames []string) error {
+	switch status {
+	case BitrotOff:
+		return nil
+	case BitrotInvalid:
+		return fmt.Errorf("bitrot sidecar is malformed or unverifiable")
+	}
+	for shardId, shardPath := range shardFileNames {
+		entry := shardChecksums(prot, uint32(shardId))
+		if entry == nil {
+			continue
+		}
+		mismatched, err := verifyShardFileBlocks(shardPath, entry, int64(prot.BlockSize))
+		if err != nil {
+			return fmt.Errorf("verify data shard %d %s: %w", shardId, shardPath, err)
+		}
+		if len(mismatched) > 0 {
+			return fmt.Errorf("data shard %d %s fails its bitrot checksum in %d blocks", shardId, shardPath, len(mismatched))
+		}
+	}
+	return nil
+}
+
 // ComputeProtectionFromShards builds a complete EcBitrotProtection by reading
 // the on-disk bytes of every shard for a generation — the trust-on-first-use
 // backfill primitive for EC volumes that were encoded before this feature. It
@@ -480,10 +526,13 @@ func (ev *EcVolume) BitrotProtection() (*volume_server_pb.EcBitrotProtection, Bi
 // already mounted — a shard delivery can bring the manifest with it, and the
 // receive path only writes the file, so without this the in-memory volume
 // keeps the protection state it resolved at mount (off) until a remount.
-func (ev *EcVolume) ReloadBitrotSidecar(additionalDirs ...string) {
-	if err := ev.loadActiveBitrotSidecar(additionalDirs...); err != nil {
+// It returns the layout-mismatch error a mount would fail on.
+func (ev *EcVolume) ReloadBitrotSidecar(additionalDirs ...string) error {
+	err := ev.loadActiveBitrotSidecar(additionalDirs...)
+	if err != nil {
 		glog.Warningf("ec volume %d: reload bitrot sidecar: %v", ev.VolumeId, err)
 	}
+	return err
 }
 
 // loadActiveBitrotSidecar loads the generation-0 checksum sidecar into the

weed/storage/erasure_coding/ec_bitrot_test.go

@@ -346,6 +346,11 @@ func TestRemoveBitrotSidecars(t *testing.T) {
 }
 
 func corruptOneByte(t *testing.T, path string) {
+	t.Helper()
+	corruptOneByteAt(t, path, 0)
+}
+
+func corruptOneByteAt(t *testing.T, path string, offset int64) {
 	t.Helper()
 	f, err := os.OpenFile(path, os.O_RDWR, 0644)
 	if err != nil {
@@ -353,11 +358,70 @@ func corruptOneByte(t *testing.T, path string) {
 	}
 	defer f.Close()
 	var b [1]byte
-	if _, err := f.ReadAt(b[:], 0); err != nil {
+	if _, err := f.ReadAt(b[:], offset); err != nil {
 		t.Fatal(err)
 	}
 	b[0] ^= 0xff
-	if _, err := f.WriteAt(b[:], 0); err != nil {
+	if _, err := f.WriteAt(b[:], offset); err != nil {
 		t.Fatal(err)
 	}
 }
+
+func TestVerifyEcVolumeDataShards(t *testing.T) {
+	old := BitrotBlockSize
+	BitrotBlockSize = 1 << 20
+	t.Cleanup(func() { BitrotBlockSize = old })
+
+	dir := t.TempDir()
+	base := filepath.Join(dir, "42")
+	writeRandomDat(t, base, 25*1024*1024)
+	ctx := &ECContext{DataShards: 10, ParityShards: 4}
+	prot, err := WriteEcFiles(base, ctx)
+	if err != nil {
+		t.Fatalf("encode: %v", err)
+	}
+
+	shardFileNames := make([]string, ctx.DataShards)
+	for id := range shardFileNames {
+		shardFileNames[id] = base + ctx.ToExt(id)
+	}
+
+	if err := VerifyEcVolumeDataShards(prot, BitrotOn, shardFileNames); err != nil {
+		t.Fatalf("clean data shards: %v", err)
+	}
+	if err := VerifyEcVolumeDataShards(nil, BitrotInvalid, shardFileNames); err == nil {
+		t.Fatal("an invalid sidecar must fail the verification")
+	}
+
+	corruptOneByteAt(t, shardFileNames[3], 1<<20)
+	if err := VerifyEcVolumeDataShards(prot, BitrotOn, shardFileNames); err == nil {
+		t.Fatal("a corrupt second block of data shard 3 must fail the verification")
+	}
+	if err := VerifyEcVolumeDataShards(nil, BitrotOff, shardFileNames); err != nil {
+		t.Fatalf("an unprotected volume must pass: %v", err)
+	}
+}
+
+func TestSelectBitrotProtection(t *testing.T) {
+	prot := &volume_server_pb.EcBitrotProtection{BlockSize: 1}
+	runtime := func(p *volume_server_pb.EcBitrotProtection, status BitrotStatus) *EcVolume {
+		return &EcVolume{bitrot: p, bitrotStatus: status}
+	}
+	for _, tc := range []struct {
+		name     string
+		volumes  []*EcVolume
+		wantProt bool
+		want     BitrotStatus
+	}{
+		{"none", nil, false, BitrotOff},
+		{"all off", []*EcVolume{runtime(nil, BitrotOff), runtime(nil, BitrotOff)}, false, BitrotOff},
+		{"sidecar on the second disk", []*EcVolume{runtime(nil, BitrotOff), runtime(prot, BitrotOn)}, true, BitrotOn},
+		{"invalid beside off", []*EcVolume{runtime(nil, BitrotOff), runtime(nil, BitrotInvalid)}, false, BitrotInvalid},
+		{"on wins over invalid", []*EcVolume{runtime(nil, BitrotInvalid), runtime(prot, BitrotOn)}, true, BitrotOn},
+	} {
+		got, status := SelectBitrotProtection(tc.volumes)
+		if status != tc.want || (got != nil) != tc.wantProt {
+			t.Errorf("%s: got (%v, %v), want status %v", tc.name, got, status, tc.want)
+		}
+	}
+}

weed/storage/erasure_coding/ec_volume_sidecar_layout_test.go

@@ -3,6 +3,7 @@ package erasure_coding
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
@@ -59,7 +60,7 @@ func TestNewEcVolumeTakesLayoutFromSidecarWhenVifHasNoConfig(t *testing.T) {
 // legacy layout.
 func TestNewEcVolumeFindsSidecarInTheIndexDirectory(t *testing.T) {
 	for _, tc := range []struct {
-		name    string
+		name     string
 		writeVif bool
 	}{
 		{name: "no vif at all"},
@@ -199,6 +200,16 @@ func TestReloadBitrotSidecarFindsItOnASiblingDisk(t *testing.T) {
 	if got := ev.bitrotStatus; got != BitrotOn {
 		t.Errorf("cross-disk reload: status = %v, want BitrotOn", got)
 	}
+
+	// A delivered sidecar that records another layout is reported, not
+	// swallowed: the decode refuses on it.
+	prot.EcShardConfig.ParityShards = 2
+	if err := SaveBitrotSidecar(BitrotSidecarPath(filepath.Join(sibling, "7"), 0), prot); err != nil {
+		t.Fatalf("seed mismatched .ecsum: %v", err)
+	}
+	if err := ev.ReloadBitrotSidecar(mine, sibling); err == nil || !strings.Contains(err.Error(), "refusing to serve") {
+		t.Errorf("mismatched-layout reload: err = %v, want the layout mismatch", err)
+	}
 }
 
 // "Does this volume already have a manifest" cannot be answered from one base