mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-11 00:50:43 +02:00
Chris Lu
d4548376a1
* fix(ec): off-by-one in nLargeBlockRows causes EC read corruption (#8947) The nLargeBlockRows formula in locateOffset used (shardDatSize-1)/largeBlockLength, which produces an off-by-one error when shardDatSize is an exact multiple of largeBlockLength (e.g. a 30GB volume with 10 data shards = 3GB per shard). This causes needles in the last large block row to be mislocated as small blocks, reading from completely wrong shard positions and returning garbage data. Fix: remove the -1 from locateOffset and only apply it in the ecdFileSize fallback path (old volumes without datFileSize in .vif), where it's needed to handle the ambiguous case conservatively. Also fix ReadEcShardNeedle to pass offset=0 to ReadBytes, consistent with the scrub path, since the bytes buffer already starts at position 0. * fix: add volume context to EC read errors, remove contextless glog The glog.Errorf in ReadBytes logged "entry not found" without any volume ID, making it impossible to identify which volume was affected. Remove this contextless log and instead add volume ID, needle ID, offset, and size to the error returned from the EC read path. The EC scrub callers already wrap errors with volume context.
99 lines
3.2 KiB
Go
99 lines
3.2 KiB
Go
package erasure_coding
|
|
|
|
import (
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
)
|
|
|
|
type Interval struct {
|
|
BlockIndex int // the index of the block in either the large blocks or the small blocks
|
|
InnerBlockOffset int64
|
|
Size types.Size
|
|
IsLargeBlock bool // whether the block is a large block or a small block
|
|
LargeBlockRowsCount int
|
|
}
|
|
|
|
// LocateData finds the intervals of data within erasure coding blocks for a given offset and size.
|
|
func LocateData(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64, size types.Size) (intervals []Interval) {
|
|
blockIndex, isLargeBlock, nLargeBlockRows, innerBlockOffset := locateOffset(largeBlockLength, smallBlockLength, shardDatSize, offset)
|
|
|
|
for size > 0 {
|
|
blockRemaining := largeBlockLength - innerBlockOffset
|
|
if !isLargeBlock {
|
|
blockRemaining = smallBlockLength - innerBlockOffset
|
|
}
|
|
|
|
if blockRemaining <= 0 {
|
|
// move to next block
|
|
blockIndex, isLargeBlock = moveToNextBlock(blockIndex, isLargeBlock, nLargeBlockRows)
|
|
innerBlockOffset = 0
|
|
continue
|
|
}
|
|
|
|
interval := Interval{
|
|
BlockIndex: blockIndex,
|
|
InnerBlockOffset: innerBlockOffset,
|
|
IsLargeBlock: isLargeBlock,
|
|
LargeBlockRowsCount: int(nLargeBlockRows),
|
|
}
|
|
|
|
if int64(size) <= blockRemaining {
|
|
interval.Size = size
|
|
intervals = append(intervals, interval)
|
|
return
|
|
}
|
|
interval.Size = types.Size(blockRemaining)
|
|
intervals = append(intervals, interval)
|
|
|
|
size -= interval.Size
|
|
blockIndex, isLargeBlock = moveToNextBlock(blockIndex, isLargeBlock, nLargeBlockRows)
|
|
innerBlockOffset = 0
|
|
|
|
}
|
|
return
|
|
}
|
|
|
|
func moveToNextBlock(blockIndex int, isLargeBlock bool, nLargeBlockRows int64) (int, bool) {
|
|
nextBlockIndex := blockIndex + 1
|
|
nextIsLargeBlock := isLargeBlock
|
|
if isLargeBlock && int64(nextBlockIndex) == nLargeBlockRows*DataShardsCount {
|
|
nextIsLargeBlock = false
|
|
nextBlockIndex = 0
|
|
}
|
|
return nextBlockIndex, nextIsLargeBlock
|
|
}
|
|
|
|
func locateOffset(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64) (blockIndex int, isLargeBlock bool, nLargeBlockRows int64, innerBlockOffset int64) {
|
|
largeRowSize := largeBlockLength * DataShardsCount
|
|
nLargeBlockRows = shardDatSize / largeBlockLength
|
|
|
|
// if offset is within the large block area
|
|
if offset < nLargeBlockRows*largeRowSize {
|
|
isLargeBlock = true
|
|
blockIndex, innerBlockOffset = locateOffsetWithinBlocks(largeBlockLength, offset)
|
|
return
|
|
}
|
|
|
|
isLargeBlock = false
|
|
offset -= nLargeBlockRows * largeRowSize
|
|
blockIndex, innerBlockOffset = locateOffsetWithinBlocks(smallBlockLength, offset)
|
|
return
|
|
}
|
|
|
|
func locateOffsetWithinBlocks(blockLength int64, offset int64) (blockIndex int, innerBlockOffset int64) {
|
|
blockIndex = int(offset / blockLength)
|
|
innerBlockOffset = offset % blockLength
|
|
return
|
|
}
|
|
|
|
func (interval Interval) ToShardIdAndOffset(largeBlockSize, smallBlockSize int64) (ShardId, int64) {
|
|
ecFileOffset := interval.InnerBlockOffset
|
|
rowIndex := interval.BlockIndex / DataShardsCount
|
|
if interval.IsLargeBlock {
|
|
ecFileOffset += int64(rowIndex) * largeBlockSize
|
|
} else {
|
|
ecFileOffset += int64(interval.LargeBlockRowsCount)*largeBlockSize + int64(rowIndex)*smallBlockSize
|
|
}
|
|
ecFileIndex := interval.BlockIndex % DataShardsCount
|
|
return ShardId(ecFileIndex), ecFileOffset
|
|
}
|