diff --git a/seaweed-volume/src/server/grpc_server.rs b/seaweed-volume/src/server/grpc_server.rs index 5ee4890416..bde0994e5c 100644 --- a/seaweed-volume/src/server/grpc_server.rs +++ b/seaweed-volume/src/server/grpc_server.rs @@ -4353,7 +4353,7 @@ impl VolumeServer for VolumeGrpcService { // however often it reloaded. One delivered copy, reachable from all. let ec_metadata_dirs = store.ec_metadata_dirs(); for ec_vol in store.find_all_ec_volumes_mut(vid) { - ec_vol.reload_bitrot_sidecar(&ec_metadata_dirs); + let _ = ec_vol.reload_bitrot_sidecar(&ec_metadata_dirs); } drop(store); self.state.volume_state_notify.notify_one(); @@ -7766,6 +7766,45 @@ impl EcDecodeJob { ))); } + let shard_paths: Vec = shard_dirs + .iter() + .enumerate() + .map(|(i, dir)| { + crate::storage::erasure_coding::ec_shard::EcVolumeShard::new( + dir, + &collection, + vid, + i as ShardId, + ) + .file_name() + }) + .collect(); + // A rebuild's backfill writes the sidecar without reloading the + // mounted runtimes, so resolve it again the way a shard mount does. + // Any runtime left finds it on every disk; with none left (an unmount + // raced the decode) the protection would read as Off. + let (bitrot, bitrot_status) = { + let mut store = state.store.write().unwrap(); + let ec_metadata_dirs = store.ec_metadata_dirs(); + let runtimes = store.find_all_ec_volumes_mut(vid); + if runtimes.is_empty() { + return Err(Status::internal(format!( + "ec volume {} was unmounted during the decode", + vid + ))); + } + for ec_vol in runtimes { + ec_vol + .reload_bitrot_sidecar(&ec_metadata_dirs) + .map_err(|e| Status::internal(format!("ec volume {}: {}", vid, e)))?; + } + crate::storage::erasure_coding::ec_volume::EcVolume::select_bitrot_protection( + &store.find_all_ec_volumes(vid), + ) + }; + ec_bitrot::verify_ec_volume_data_shards(bitrot.as_ref(), bitrot_status, &shard_paths) + .map_err(|e| Status::internal(format!("ec volume {}: {}", vid, e)))?; + // Calculate .dat file size from .ecx entries (.ec00 lives on // its own disk, .ecx on the index disk). let dat_file_size = ec_decoder::find_dat_file_size_with_dirs( @@ -14805,6 +14844,149 @@ mod tests { assert!(std::path::Path::new(&idx_sidecar).exists()); } + /// A data shard that fails its .ecsum checksum must fail the decode and + /// leave every shard and the sidecar in place for a repair. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_volume_ec_shards_to_volume_rejects_a_corrupt_data_shard() { + let (service, _tmp, data, _idx, _) = make_split_idx_ec_decode_service(&[], false).await; + assert_corrupt_data_shard_rejected(&service, &data).await; + } + + /// A rebuild's backfill writes the sidecar after the shards are mounted, + /// without reloading the runtimes: the decode must still find it. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_volume_ec_shards_to_volume_rejects_a_corrupt_shard_with_a_late_sidecar() { + let (service, _tmp, data, _idx, _) = make_split_idx_ec_decode_service(&[], false).await; + let sidecar = format!("{data}/1.ecsum"); + let hidden = format!("{data}/1.ecsum.hidden"); + std::fs::rename(&sidecar, &hidden).unwrap(); + { + let mut store = service.state.store.write().unwrap(); + for ec_vol in store.find_all_ec_volumes_mut(VolumeId(1)) { + ec_vol.reload_bitrot_sidecar(&[]).unwrap(); + assert_eq!( + ec_vol.bitrot_protection().1, + crate::storage::erasure_coding::ec_bitrot::BitrotStatus::Off, + "precondition: the mounted runtime holds no protection" + ); + } + } + std::fs::rename(&hidden, &sidecar).unwrap(); + assert_corrupt_data_shard_rejected(&service, &data).await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_volume_ec_shards_to_volume_refuses_a_late_sidecar_with_another_layout() { + use crate::storage::erasure_coding::ec_bitrot; + let (service, _tmp, data, _idx, _) = make_split_idx_ec_decode_service(&[], false).await; + let sidecar = format!("{data}/1.ecsum"); + let hidden = format!("{data}/1.ecsum.hidden"); + std::fs::rename(&sidecar, &hidden).unwrap(); + { + let mut store = service.state.store.write().unwrap(); + for ec_vol in store.find_all_ec_volumes_mut(VolumeId(1)) { + ec_vol.reload_bitrot_sidecar(&[]).unwrap(); + } + } + let mut prot = ec_bitrot::load_bitrot_sidecar(&hidden).unwrap(); + prot.ec_shard_config.as_mut().unwrap().parity_shards = 2; + ec_bitrot::save_bitrot_sidecar(&sidecar, &prot).unwrap(); + std::fs::remove_file(&hidden).unwrap(); + + let err = service + .volume_ec_shards_to_volume(ec_shards_to_volume_request()) + .await + .unwrap_err(); + + assert!( + err.message().contains("refusing to serve"), + "unexpected error: {}", + err.message() + ); + assert!(!std::path::Path::new(&format!("{data}/1.dat")).exists()); + for shard_id in 0..14 { + assert!(std::path::Path::new(&format!("{data}/1.ec{shard_id:02}")).exists()); + } + } + + /// An unmount between the job snapshot and the verification leaves no + /// runtime to resolve the sidecar: the job must refuse, not verify as Off. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_ec_decode_job_refuses_when_every_runtime_was_unmounted() { + let (service, _tmp, data, _idx, _) = make_split_idx_ec_decode_service(&[], false).await; + let job = { + let store = service.state.store.read().unwrap(); + let (ec_vol, shard_dirs) = store + .collect_ec_shard_dirs( + VolumeId(1), + crate::storage::erasure_coding::ec_shard::MAX_SHARD_COUNT, + ) + .unwrap(); + let data_shards = ec_vol.data_shards as usize; + EcDecodeJob { + vid: VolumeId(1), + dat_dir: ec_vol.dir.clone(), + ecx_dir: ec_vol.ecx_actual_dir().to_string(), + idx_dir: ec_vol.dir_idx.clone(), + collection: ec_vol.collection.clone(), + vif_dat_file_size: ec_vol.dat_file_size, + large_block_size: ec_vol.large_block_size() as usize, + small_block_size: ec_vol.small_block_size() as usize, + shard_dirs: shard_dirs[..data_shards] + .iter() + .map(|d| d.clone().unwrap()) + .collect(), + needle_map_kind: store.needle_map_kind, + } + }; + service + .volume_ec_shards_unmount(Request::new( + volume_server_pb::VolumeEcShardsUnmountRequest { + volume_id: 1, + shard_ids: (0..14).collect(), + encode_ts_ns: 0, + }, + )) + .await + .unwrap(); + let shard = format!("{data}/1.ec03"); + let mut bytes = std::fs::read(&shard).unwrap(); + bytes[0] ^= 0xff; + std::fs::write(&shard, bytes).unwrap(); + + let err = job.run(&service.state).unwrap_err(); + + assert!( + err.message().contains("unmounted during the decode"), + "unexpected error: {}", + err.message() + ); + assert!(!std::path::Path::new(&format!("{data}/1.dat")).exists()); + } + + async fn assert_corrupt_data_shard_rejected(service: &VolumeGrpcService, data: &str) { + let shard = format!("{data}/1.ec03"); + let mut bytes = std::fs::read(&shard).unwrap(); + bytes[0] ^= 0xff; + std::fs::write(&shard, bytes).unwrap(); + + let err = service + .volume_ec_shards_to_volume(ec_shards_to_volume_request()) + .await + .unwrap_err(); + + assert!( + err.message().contains("data shard 3"), + "unexpected error: {}", + err.message() + ); + assert!(!std::path::Path::new(&format!("{data}/1.dat")).exists()); + assert!(std::path::Path::new(&format!("{data}/1.ecsum")).exists()); + for shard_id in 0..14 { + assert!(std::path::Path::new(&format!("{data}/1.ec{shard_id:02}")).exists()); + } + } + /// Replaces the idx-dir .ecj with a FIFO: the decode's open of it blocks /// until [`release_ecj_fifo`] opens the write end. #[cfg(unix)] diff --git a/seaweed-volume/src/storage/erasure_coding/ec_bitrot.rs b/seaweed-volume/src/storage/erasure_coding/ec_bitrot.rs index 7c7d44549f..755f1499e0 100644 --- a/seaweed-volume/src/storage/erasure_coding/ec_bitrot.rs +++ b/seaweed-volume/src/storage/erasure_coding/ec_bitrot.rs @@ -581,6 +581,51 @@ pub fn verify_shard_blocks( Ok(mismatched) } +/// Checks the data shards a decode reads against the volume's bitrot +/// protection: a corrupt shard copied into the .dat is lost for good once the +/// caller deletes the parity that could repair it. Mirrors Go's +/// `VerifyEcVolumeDataShards`. +pub fn verify_ec_volume_data_shards( + prot: Option<&EcBitrotProtection>, + status: BitrotStatus, + shard_paths: &[String], +) -> io::Result<()> { + let prot = match (status, prot) { + (BitrotStatus::Off, _) => return Ok(()), + (BitrotStatus::On, Some(prot)) => prot, + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "bitrot sidecar is malformed or unverifiable", + )); + } + }; + for (shard_id, path) in shard_paths.iter().enumerate() { + let Some(entry) = shard_checksums(prot, shard_id as u32) else { + continue; + }; + let mismatched = + verify_shard_file_blocks(path, entry, prot.block_size as i64).map_err(|e| { + io::Error::new( + e.kind(), + format!("verify data shard {} {}: {}", shard_id, path, e), + ) + })?; + if !mismatched.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "data shard {} {} fails its bitrot checksum in {} blocks", + shard_id, + path, + mismatched.len() + ), + )); + } + } + Ok(()) +} + /// Builds the `EcShardConfig` proto for the given layout. The bitrot sidecar /// carries its own top-level encode_uuid, so the nested config leaves it empty. pub fn ec_shard_config(data_shards: u32, parity_shards: u32, block_size: i64) -> EcShardConfig { diff --git a/seaweed-volume/src/storage/erasure_coding/ec_volume.rs b/seaweed-volume/src/storage/erasure_coding/ec_volume.rs index da9598f6c0..29433f052c 100644 --- a/seaweed-volume/src/storage/erasure_coding/ec_volume.rs +++ b/seaweed-volume/src/storage/erasure_coding/ec_volume.rs @@ -238,8 +238,8 @@ pub struct EcVolume { /// no encode identity, so provenance is the only signal that a loaded /// manifest may describe a different encode run. Same purpose as /// `ecx_actual_dir`. Empty when no sidecar was found. Always committed in - /// the same step as `bitrot`/`bitrot_status`, so a reload whose `Err` gets - /// swallowed by `reload_bitrot_sidecar` cannot leave this describing a + /// the same step as `bitrot`/`bitrot_status`, so a reload whose `Err` a + /// caller ignores cannot leave this describing a /// sidecar other than the one those two fields actually hold. pub(crate) bitrot_source_dir: String, @@ -802,14 +802,17 @@ impl EcVolume { /// shard delivery can bring the manifest with it, and the receive path only /// writes the file, so without this the in-memory volume keeps the /// protection state it resolved at mount (off) until a remount. - pub fn reload_bitrot_sidecar(&mut self, additional_dirs: &[String]) { - if let Err(e) = self.load_active_bitrot_sidecar(additional_dirs) { + /// Returns the layout-mismatch error a mount would fail on. + pub fn reload_bitrot_sidecar(&mut self, additional_dirs: &[String]) -> io::Result<()> { + let result = self.load_active_bitrot_sidecar(additional_dirs); + if let Err(e) = &result { tracing::warn!( volume_id = self.volume_id.0, error = %e, "reload bitrot sidecar", ); } + result } /// Load the generation-0 checksum sidecar into `self.bitrot`/`self.bitrot_status`. @@ -888,8 +891,8 @@ impl EcVolume { self.parity_shards as usize, ); // Provenance is committed together with the sidecar it describes, so a - // swallowed reload (`reload_bitrot_sidecar` logs and discards the `Err` - // above) cannot leave them disagreeing: the geometry-mismatch return + // reload whose `Err` above a caller ignores cannot leave them + // disagreeing: the geometry-mismatch return // happens before this point, so `bitrot`/`bitrot_status` below and // `bitrot_source_dir` here always describe the same load attempt. // Only a path that actually exists is a real source: `path` falls back @@ -934,6 +937,31 @@ impl EcVolume { (self.bitrot.clone(), self.bitrot_status) } + /// The protection of a volume's per-disk runtimes: the first `On`, else + /// `Invalid`, else `Off`. After a restart only the runtime whose disk holds + /// the sidecar resolves it. Mirrors Go's `SelectBitrotProtection`. + pub(crate) fn select_bitrot_protection( + volumes: &[&EcVolume], + ) -> ( + Option, + crate::storage::erasure_coding::ec_bitrot::BitrotStatus, + ) { + use crate::storage::erasure_coding::ec_bitrot::BitrotStatus; + let mut invalid = false; + for v in volumes { + match v.bitrot_protection() { + (prot, BitrotStatus::On) => return (prot, BitrotStatus::On), + (_, BitrotStatus::Invalid) => invalid = true, + _ => {} + } + } + if invalid { + (None, BitrotStatus::Invalid) + } else { + (None, BitrotStatus::Off) + } + } + /// Read-only EC bitrot checksum scrub of the LOCAL shards of this volume's /// active generation. /// diff --git a/test/volume_server/grpc/ec_decode_bitrot_test.go b/test/volume_server/grpc/ec_decode_bitrot_test.go new file mode 100644 index 0000000000..d2a2b0cc6b --- /dev/null +++ b/test/volume_server/grpc/ec_decode_bitrot_test.go @@ -0,0 +1,125 @@ +package volume_server_grpc_test + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/test/volume_server/framework" + "github.com/seaweedfs/seaweedfs/test/volume_server/matrix" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" +) + +func TestEcShardsToVolumeRejectsCorruptDataShard(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + t.Run("sidecar loaded at mount", func(t *testing.T) { + testEcShardsToVolumeRejectsCorruptDataShard(t, false, 0, "data shard 3") + }) + // A rebuild's backfill writes the sidecar after the shards are mounted. + t.Run("sidecar written after mount", func(t *testing.T) { + testEcShardsToVolumeRejectsCorruptDataShard(t, true, 0, "data shard 3") + }) + t.Run("sidecar with another layout written after mount", func(t *testing.T) { + testEcShardsToVolumeRejectsCorruptDataShard(t, true, 2, "refusing to serve") + }) +} + +func testEcShardsToVolumeRejectsCorruptDataShard(t *testing.T, sidecarAfterMount bool, lateParityShards uint32, wantErr string) { + clusterHarness := framework.StartVolumeCluster(t, matrix.P1()) + conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress()) + defer conn.Close() + + const volumeID = uint32(123) + framework.AllocateVolume(t, grpcClient, volumeID, "") + + httpClient := framework.NewHTTPClient() + fid := framework.NewFileID(volumeID, 990007, 0x66771123) + payload := []byte("ec-shards-to-volume-corrupt-data-shard") + uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, payload) + _ = framework.ReadAllAndClose(t, uploadResp) + if uploadResp.StatusCode != http.StatusCreated { + t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: volumeID, + }); err != nil { + t.Fatalf("VolumeEcShardsGenerate: %v", err) + } + // Reads must go through the EC shards, not the source volume. + if _, err := grpcClient.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{ + VolumeId: volumeID, + }); err != nil { + t.Fatalf("VolumeUnmount: %v", err) + } + dataDir := filepath.Join(clusterHarness.BaseDir(), "volume") + base := filepath.Join(dataDir, fmt.Sprintf("%d", volumeID)) + if _, err := os.Stat(base + ".ecsum"); err != nil { + t.Fatalf("precondition: the encode wrote a bitrot sidecar: %v", err) + } + if sidecarAfterMount { + if err := os.Rename(base+".ecsum", base+".ecsum.hidden"); err != nil { + t.Fatalf("hide sidecar: %v", err) + } + } + if _, err := grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: volumeID, + ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, + }); err != nil { + t.Fatalf("VolumeEcShardsMount: %v", err) + } + if sidecarAfterMount { + if err := os.Rename(base+".ecsum.hidden", base+".ecsum"); err != nil { + t.Fatalf("restore sidecar: %v", err) + } + } + if lateParityShards != 0 { + prot, err := erasure_coding.LoadBitrotSidecar(base + ".ecsum") + if err != nil { + t.Fatalf("load sidecar: %v", err) + } + prot.EcShardConfig.ParityShards = lateParityShards + if err := erasure_coding.SaveBitrotSidecar(base+".ecsum", prot); err != nil { + t.Fatalf("rewrite sidecar: %v", err) + } + } + shard := base + ".ec03" + content, err := os.ReadFile(shard) + if err != nil { + t.Fatalf("read %s: %v", shard, err) + } + content[0] ^= 0xff + if err := os.WriteFile(shard, content, 0644); err != nil { + t.Fatalf("corrupt %s: %v", shard, err) + } + + _, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{ + VolumeId: volumeID, + }) + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("VolumeEcShardsToVolume with a corrupt data shard: %v", err) + } + + for _, ext := range []string{".ecsum", ".ec00", ".ec03", ".ec13"} { + if _, statErr := os.Stat(base + ext); statErr != nil { + t.Fatalf("%s must survive the refused decode: %v", ext, statErr) + } + } + + readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid) + readBody := framework.ReadAllAndClose(t, readResp) + if readResp.StatusCode != http.StatusOK || string(readBody) != string(payload) { + t.Fatalf("EC read after the refused decode: status %d body %q", readResp.StatusCode, readBody) + } +} diff --git a/weed/server/volume_grpc_erasure_coding.go b/weed/server/volume_grpc_erasure_coding.go index 86842309e3..22073ea64b 100644 --- a/weed/server/volume_grpc_erasure_coding.go +++ b/weed/server/volume_grpc_erasure_coding.go @@ -1302,6 +1302,25 @@ func (vs *VolumeServer) VolumeEcShardsToVolume(ctx context.Context, req *volume_ return nil, status.Errorf(codes.FailedPrecondition, "ec volume %d %s", req.VolumeId, erasure_coding.EcNoLiveEntriesSubstring) } + // A rebuild's backfill writes the sidecar without reloading the mounted + // runtimes, so resolve it again the way a shard mount does. Any runtime + // left finds it on every disk; with none left (an unmount raced the + // decode) the protection would read as Off. + ecVolumes := vs.store.FindAllEcVolumes(needle.VolumeId(req.VolumeId)) + if len(ecVolumes) == 0 { + return nil, fmt.Errorf("ec volume %d was unmounted during the decode", req.VolumeId) + } + ecMetadataDirs := vs.store.EcMetadataDirs() + for _, ecVolume := range ecVolumes { + if err := ecVolume.ReloadBitrotSidecar(ecMetadataDirs...); err != nil { + return nil, fmt.Errorf("ec volume %d: %w", req.VolumeId, err) + } + } + prot, bitrotStatus := erasure_coding.SelectBitrotProtection(ecVolumes) + if err := erasure_coding.VerifyEcVolumeDataShards(prot, bitrotStatus, shardFileNames); err != nil { + return nil, fmt.Errorf("ec volume %d: %w", req.VolumeId, err) + } + // calculate .dat file size. Pass shard 0's resolved path: on a multi-disk // server the volume's shards can sit on several disks, so the .ec00 is not // necessarily beside the EcVolume's own base path. diff --git a/weed/storage/erasure_coding/ec_bitrot.go b/weed/storage/erasure_coding/ec_bitrot.go index dfe78a5f2c..2255c13547 100644 --- a/weed/storage/erasure_coding/ec_bitrot.go +++ b/weed/storage/erasure_coding/ec_bitrot.go @@ -399,6 +399,52 @@ func verifyShardFileBlocks(path string, entry *volume_server_pb.EcShardChecksums return mismatched, nil } +// SelectBitrotProtection picks the protection of a volume's per-disk runtimes: +// the first On, else the first Invalid, else Off. After a restart only the +// runtime whose disk holds the sidecar resolves it. +func SelectBitrotProtection(volumes []*EcVolume) (*volume_server_pb.EcBitrotProtection, BitrotStatus) { + var invalid bool + for _, v := range volumes { + prot, status := v.BitrotProtection() + switch status { + case BitrotOn: + return prot, status + case BitrotInvalid: + invalid = true + } + } + if invalid { + return nil, BitrotInvalid + } + return nil, BitrotOff +} + +// VerifyEcVolumeDataShards checks the data shards a decode reads against the +// volume's bitrot protection: a corrupt shard copied into the .dat is lost for +// good once the caller deletes the parity that could repair it. +func VerifyEcVolumeDataShards(prot *volume_server_pb.EcBitrotProtection, status BitrotStatus, shardFileNames []string) error { + switch status { + case BitrotOff: + return nil + case BitrotInvalid: + return fmt.Errorf("bitrot sidecar is malformed or unverifiable") + } + for shardId, shardPath := range shardFileNames { + entry := shardChecksums(prot, uint32(shardId)) + if entry == nil { + continue + } + mismatched, err := verifyShardFileBlocks(shardPath, entry, int64(prot.BlockSize)) + if err != nil { + return fmt.Errorf("verify data shard %d %s: %w", shardId, shardPath, err) + } + if len(mismatched) > 0 { + return fmt.Errorf("data shard %d %s fails its bitrot checksum in %d blocks", shardId, shardPath, len(mismatched)) + } + } + return nil +} + // ComputeProtectionFromShards builds a complete EcBitrotProtection by reading // the on-disk bytes of every shard for a generation — the trust-on-first-use // backfill primitive for EC volumes that were encoded before this feature. It @@ -480,10 +526,13 @@ func (ev *EcVolume) BitrotProtection() (*volume_server_pb.EcBitrotProtection, Bi // already mounted — a shard delivery can bring the manifest with it, and the // receive path only writes the file, so without this the in-memory volume // keeps the protection state it resolved at mount (off) until a remount. -func (ev *EcVolume) ReloadBitrotSidecar(additionalDirs ...string) { - if err := ev.loadActiveBitrotSidecar(additionalDirs...); err != nil { +// It returns the layout-mismatch error a mount would fail on. +func (ev *EcVolume) ReloadBitrotSidecar(additionalDirs ...string) error { + err := ev.loadActiveBitrotSidecar(additionalDirs...) + if err != nil { glog.Warningf("ec volume %d: reload bitrot sidecar: %v", ev.VolumeId, err) } + return err } // loadActiveBitrotSidecar loads the generation-0 checksum sidecar into the diff --git a/weed/storage/erasure_coding/ec_bitrot_test.go b/weed/storage/erasure_coding/ec_bitrot_test.go index b8cf6758a8..299274a588 100644 --- a/weed/storage/erasure_coding/ec_bitrot_test.go +++ b/weed/storage/erasure_coding/ec_bitrot_test.go @@ -346,6 +346,11 @@ func TestRemoveBitrotSidecars(t *testing.T) { } func corruptOneByte(t *testing.T, path string) { + t.Helper() + corruptOneByteAt(t, path, 0) +} + +func corruptOneByteAt(t *testing.T, path string, offset int64) { t.Helper() f, err := os.OpenFile(path, os.O_RDWR, 0644) if err != nil { @@ -353,11 +358,70 @@ func corruptOneByte(t *testing.T, path string) { } defer f.Close() var b [1]byte - if _, err := f.ReadAt(b[:], 0); err != nil { + if _, err := f.ReadAt(b[:], offset); err != nil { t.Fatal(err) } b[0] ^= 0xff - if _, err := f.WriteAt(b[:], 0); err != nil { + if _, err := f.WriteAt(b[:], offset); err != nil { t.Fatal(err) } } + +func TestVerifyEcVolumeDataShards(t *testing.T) { + old := BitrotBlockSize + BitrotBlockSize = 1 << 20 + t.Cleanup(func() { BitrotBlockSize = old }) + + dir := t.TempDir() + base := filepath.Join(dir, "42") + writeRandomDat(t, base, 25*1024*1024) + ctx := &ECContext{DataShards: 10, ParityShards: 4} + prot, err := WriteEcFiles(base, ctx) + if err != nil { + t.Fatalf("encode: %v", err) + } + + shardFileNames := make([]string, ctx.DataShards) + for id := range shardFileNames { + shardFileNames[id] = base + ctx.ToExt(id) + } + + if err := VerifyEcVolumeDataShards(prot, BitrotOn, shardFileNames); err != nil { + t.Fatalf("clean data shards: %v", err) + } + if err := VerifyEcVolumeDataShards(nil, BitrotInvalid, shardFileNames); err == nil { + t.Fatal("an invalid sidecar must fail the verification") + } + + corruptOneByteAt(t, shardFileNames[3], 1<<20) + if err := VerifyEcVolumeDataShards(prot, BitrotOn, shardFileNames); err == nil { + t.Fatal("a corrupt second block of data shard 3 must fail the verification") + } + if err := VerifyEcVolumeDataShards(nil, BitrotOff, shardFileNames); err != nil { + t.Fatalf("an unprotected volume must pass: %v", err) + } +} + +func TestSelectBitrotProtection(t *testing.T) { + prot := &volume_server_pb.EcBitrotProtection{BlockSize: 1} + runtime := func(p *volume_server_pb.EcBitrotProtection, status BitrotStatus) *EcVolume { + return &EcVolume{bitrot: p, bitrotStatus: status} + } + for _, tc := range []struct { + name string + volumes []*EcVolume + wantProt bool + want BitrotStatus + }{ + {"none", nil, false, BitrotOff}, + {"all off", []*EcVolume{runtime(nil, BitrotOff), runtime(nil, BitrotOff)}, false, BitrotOff}, + {"sidecar on the second disk", []*EcVolume{runtime(nil, BitrotOff), runtime(prot, BitrotOn)}, true, BitrotOn}, + {"invalid beside off", []*EcVolume{runtime(nil, BitrotOff), runtime(nil, BitrotInvalid)}, false, BitrotInvalid}, + {"on wins over invalid", []*EcVolume{runtime(nil, BitrotInvalid), runtime(prot, BitrotOn)}, true, BitrotOn}, + } { + got, status := SelectBitrotProtection(tc.volumes) + if status != tc.want || (got != nil) != tc.wantProt { + t.Errorf("%s: got (%v, %v), want status %v", tc.name, got, status, tc.want) + } + } +} diff --git a/weed/storage/erasure_coding/ec_volume_sidecar_layout_test.go b/weed/storage/erasure_coding/ec_volume_sidecar_layout_test.go index 34e992416e..847006f49a 100644 --- a/weed/storage/erasure_coding/ec_volume_sidecar_layout_test.go +++ b/weed/storage/erasure_coding/ec_volume_sidecar_layout_test.go @@ -3,6 +3,7 @@ package erasure_coding import ( "os" "path/filepath" + "strings" "testing" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" @@ -59,7 +60,7 @@ func TestNewEcVolumeTakesLayoutFromSidecarWhenVifHasNoConfig(t *testing.T) { // legacy layout. func TestNewEcVolumeFindsSidecarInTheIndexDirectory(t *testing.T) { for _, tc := range []struct { - name string + name string writeVif bool }{ {name: "no vif at all"}, @@ -199,6 +200,16 @@ func TestReloadBitrotSidecarFindsItOnASiblingDisk(t *testing.T) { if got := ev.bitrotStatus; got != BitrotOn { t.Errorf("cross-disk reload: status = %v, want BitrotOn", got) } + + // A delivered sidecar that records another layout is reported, not + // swallowed: the decode refuses on it. + prot.EcShardConfig.ParityShards = 2 + if err := SaveBitrotSidecar(BitrotSidecarPath(filepath.Join(sibling, "7"), 0), prot); err != nil { + t.Fatalf("seed mismatched .ecsum: %v", err) + } + if err := ev.ReloadBitrotSidecar(mine, sibling); err == nil || !strings.Contains(err.Error(), "refusing to serve") { + t.Errorf("mismatched-layout reload: err = %v, want the layout mismatch", err) + } } // "Does this volume already have a manifest" cannot be answered from one base