mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-30 08:50:48 +01:00
EC bitrot follow-ups: protect destination sidecar on optional copy; cap sidecar block_size (#9763)
* fix(ec_bitrot): cap sidecar block_size in ValidateBitrotManifest A sidecar loaded from disk (or supplied via a backfill/peer RPC) could carry a huge power-of-two block_size that passed validation, then force a multi-GiB scratch-buffer allocation in scrub/verify. Add a shared MaxBitrotBlockSize (64 MiB) constant, enforce it as an upper bound in isPow2MultipleOf1MiB, and derive the volume flag cap from the same constant so they cannot drift. * fix(ec_bitrot): don't destroy a valid destination sidecar on an optional copy writeToFile opened the destination with O_TRUNC before knowing whether the source had the file, so an optional copy (ignoreSourceFileNotFound) from a source that lacks the .ecsum truncated and then removed a valid pre-existing destination sidecar. Stage the optional copy into a temp sibling and commit it with an atomic rename only when the source actually delivered the file; a missing source is now a no-op. Mandatory copies keep their in-place behavior.
This commit is contained in:
1 parent
9658f309d2
commit
80dd3b2621
5 files changed
+113
-31
No files matched your search
@@ -47,6 +47,10 @@ const (
|
||||
// to a 16 MiB region.
|
||||
DefaultBitrotBlockSize = 16 * 1024 * 1024
|
||||
|
||||
// MaxBitrotBlockSize caps the block granularity so a loaded sidecar cannot
|
||||
// force a huge scrub/verify scratch buffer. Power-of-two multiple of 1 MiB.
|
||||
MaxBitrotBlockSize = 64 * 1024 * 1024
|
||||
|
||||
bitrotMagic uint32 = 0x45435355 // "ECSU"
|
||||
bitrotFormatVersion uint16 = 1
|
||||
bitrotHeaderSize = 14 // magic(4)+version(2)+payload_len(4)+payload_crc32c(4)
|
||||
@@ -117,10 +121,10 @@ func NewEncodeUUID() []byte {
|
||||
return b
|
||||
}
|
||||
|
||||
// isPow2MultipleOf1MiB reports whether block_size is a power-of-two multiple of
|
||||
// 1 MiB (i.e. a power of two that is at least 1 MiB).
|
||||
// isPow2MultipleOf1MiB reports whether block_size is a power of two in
|
||||
// [1 MiB, MaxBitrotBlockSize].
|
||||
func isPow2MultipleOf1MiB(blockSize uint32) bool {
|
||||
return blockSize >= (1<<20) && bits.OnesCount32(blockSize) == 1
|
||||
return blockSize >= (1<<20) && blockSize <= MaxBitrotBlockSize && bits.OnesCount32(blockSize) == 1
|
||||
}
|
||||
|
||||
// shardChecksumBuilder accumulates the per-block CRC32C of a single shard's byte
|
||||
@@ -300,7 +304,7 @@ func ValidateBitrotManifest(prot *volume_server_pb.EcBitrotProtection, dataShard
|
||||
}
|
||||
bs := int64(prot.BlockSize)
|
||||
if !isPow2MultipleOf1MiB(prot.BlockSize) {
|
||||
return fmt.Errorf("invalid block_size %d (must be a power-of-two multiple of 1 MiB)", prot.BlockSize)
|
||||
return fmt.Errorf("invalid block_size %d (must be a power-of-two multiple of 1 MiB, at most %d)", prot.BlockSize, MaxBitrotBlockSize)
|
||||
}
|
||||
total := dataShards + parityShards
|
||||
if total <= 0 || total > MaxShardCount {
|
||||
|
||||
Reference in new issue
Block a user