weed/server: verify data shards against bitrot sidecar before ec decode (#12422) (#12427)

* weed/server: verify data shards against bitrot sidecar before ec decode (#12422)

* ec.decode: verify data shards against the mounted volume's bitrot protection

The decode looked up only the generation-0 .ecsum beside the data base and
each location's data directory, so it missed a sidecar kept with the index
and ignored the generation the volume actually serves.

Take the protection the volume's per-disk runtimes resolved instead (On, else
Invalid, else Off), and test a corrupt block past the first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* ec.decode: check for live entries before verifying the shards

A volume whose needles are all deleted gets its shards purged on the
no-live-entries status; a failing checksum must not keep it around.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* seaweed-volume: verify data shards against the bitrot protection before a decode

Mirrors the Go volume server: the decode takes the protection the volume's
per-disk runtimes resolved and refuses a data shard that fails its .ecsum
checksum, after the no-live-entries check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: a decode refuses a data shard that fails its .ecsum checksum

Runs against both volume server implementations: the refused decode keeps
every shard and the sidecar, and the needle stays readable through EC.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* ec.decode: reload the bitrot sidecar before verifying the shards

A rebuild's backfill writes the sidecar without reloading the mounted
runtimes, which then still report no protection; resolve it again across
the EC metadata directories, as a shard mount does.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* seaweed-volume: reload the bitrot sidecar before verifying the shards

Mirrors the Go volume server: the decode job resolves the sidecar again on
every runtime of the volume before choosing its protection.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: a decode finds a sidecar written after the shards were mounted

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: read through the EC shards after the refused decode

The source volume stayed mounted, so the final read never touched the
shards it is meant to check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* ec.decode: refuse when a sidecar reload reports another layout

ReloadBitrotSidecar logged and dropped the layout-mismatch error, so the
runtime stayed Off and the decode skipped the data-shard check. It now
returns that error and the decode refuses on it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* seaweed-volume: refuse a decode when a sidecar reload reports another layout

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: a decode refuses a late sidecar that records another layout

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* ec.decode: refuse when an unmount left no runtime to resolve the sidecar

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* seaweed-volume: refuse a decode when an unmount left no runtime to resolve the sidecar

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

test/volume_server/grpc/ec_decode_bitrot_test.go

@@ -0,0 +1,125 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+func TestEcShardsToVolumeRejectsCorruptDataShard(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+	t.Run("sidecar loaded at mount", func(t *testing.T) {
+		testEcShardsToVolumeRejectsCorruptDataShard(t, false, 0, "data shard 3")
+	})
+	// A rebuild's backfill writes the sidecar after the shards are mounted.
+	t.Run("sidecar written after mount", func(t *testing.T) {
+		testEcShardsToVolumeRejectsCorruptDataShard(t, true, 0, "data shard 3")
+	})
+	t.Run("sidecar with another layout written after mount", func(t *testing.T) {
+		testEcShardsToVolumeRejectsCorruptDataShard(t, true, 2, "refusing to serve")
+	})
+}
+
+func testEcShardsToVolumeRejectsCorruptDataShard(t *testing.T, sidecarAfterMount bool, lateParityShards uint32, wantErr string) {
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(123)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 990007, 0x66771123)
+	payload := []byte("ec-shards-to-volume-corrupt-data-shard")
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId: volumeID,
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsGenerate: %v", err)
+	}
+	// Reads must go through the EC shards, not the source volume.
+	if _, err := grpcClient.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{
+		VolumeId: volumeID,
+	}); err != nil {
+		t.Fatalf("VolumeUnmount: %v", err)
+	}
+	dataDir := filepath.Join(clusterHarness.BaseDir(), "volume")
+	base := filepath.Join(dataDir, fmt.Sprintf("%d", volumeID))
+	if _, err := os.Stat(base + ".ecsum"); err != nil {
+		t.Fatalf("precondition: the encode wrote a bitrot sidecar: %v", err)
+	}
+	if sidecarAfterMount {
+		if err := os.Rename(base+".ecsum", base+".ecsum.hidden"); err != nil {
+			t.Fatalf("hide sidecar: %v", err)
+		}
+	}
+	if _, err := grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId: volumeID,
+		ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13},
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsMount: %v", err)
+	}
+	if sidecarAfterMount {
+		if err := os.Rename(base+".ecsum.hidden", base+".ecsum"); err != nil {
+			t.Fatalf("restore sidecar: %v", err)
+		}
+	}
+	if lateParityShards != 0 {
+		prot, err := erasure_coding.LoadBitrotSidecar(base + ".ecsum")
+		if err != nil {
+			t.Fatalf("load sidecar: %v", err)
+		}
+		prot.EcShardConfig.ParityShards = lateParityShards
+		if err := erasure_coding.SaveBitrotSidecar(base+".ecsum", prot); err != nil {
+			t.Fatalf("rewrite sidecar: %v", err)
+		}
+	}
+	shard := base + ".ec03"
+	content, err := os.ReadFile(shard)
+	if err != nil {
+		t.Fatalf("read %s: %v", shard, err)
+	}
+	content[0] ^= 0xff
+	if err := os.WriteFile(shard, content, 0644); err != nil {
+		t.Fatalf("corrupt %s: %v", shard, err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId: volumeID,
+	})
+	if err == nil || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("VolumeEcShardsToVolume with a corrupt data shard: %v", err)
+	}
+
+	for _, ext := range []string{".ecsum", ".ec00", ".ec03", ".ec13"} {
+		if _, statErr := os.Stat(base + ext); statErr != nil {
+			t.Fatalf("%s must survive the refused decode: %v", ext, statErr)
+		}
+	}
+
+	readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid)
+	readBody := framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusOK || string(readBody) != string(payload) {
+		t.Fatalf("EC read after the refused decode: status %d body %q", readResp.StatusCode, readBody)
+	}
+}