mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 19:03:54 +01:00
fix(ec): skip 0-byte residue and verify shard sizes before rebuild (#9340)
A previously aborted ec.rebuild leaves 0-byte placeholder files for the shards it was about to write. On the next attempt, findShardFile saw those ghosts as "present", the read loop returned nil on the first n==0, and rebuild claimed success while leaving the volume mounted with empty stripes — manifesting downstream as "too few shards given" or silent corruption. Now: skip 0-byte files when scanning, validate that all surviving shards agree on size, bound the read loop by that size instead of bailing on the first short read, and remove residue after a successful rebuild so it can't shadow real shards next time.
This commit is contained in:
1 parent
e96190d128
commit
b122308bf4
2 files changed
+337
-42
No files matched your search
@@ -13,7 +13,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -128,19 +127,31 @@ func generateEcFiles(baseFileName string, bufferSize int, largeBlockSize int64,
|
||||
}
|
||||
|
||||
// findShardFile looks for a shard file at baseFileName+ext, then in additionalDirs.
|
||||
func findShardFile(baseFileName string, ext string, additionalDirs []string) string {
|
||||
primary := baseFileName + ext
|
||||
if util.FileExists(primary) {
|
||||
return primary
|
||||
}
|
||||
// Returns the first non-empty shard file path found. Any 0-byte residue files
|
||||
// (typically left behind by a previously aborted rebuild) are returned in ghosts
|
||||
// so the caller can remove them — otherwise they shadow real shards on the next
|
||||
// rebuild attempt and the volume gets silently mounted with empty stripes.
|
||||
func findShardFile(baseFileName string, ext string, additionalDirs []string) (shardPath string, ghosts []string) {
|
||||
candidates := make([]string, 0, 1+len(additionalDirs))
|
||||
candidates = append(candidates, baseFileName+ext)
|
||||
baseName := filepath.Base(baseFileName)
|
||||
for _, dir := range additionalDirs {
|
||||
candidate := filepath.Join(dir, baseName+ext)
|
||||
if util.FileExists(candidate) {
|
||||
return candidate
|
||||
candidates = append(candidates, filepath.Join(dir, baseName+ext))
|
||||
}
|
||||
for _, c := range candidates {
|
||||
fi, statErr := os.Stat(c)
|
||||
if statErr != nil {
|
||||
continue
|
||||
}
|
||||
if fi.Size() == 0 {
|
||||
ghosts = append(ghosts, c)
|
||||
continue
|
||||
}
|
||||
if shardPath == "" {
|
||||
shardPath = c
|
||||
}
|
||||
}
|
||||
return ""
|
||||
return
|
||||
}
|
||||
|
||||
func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize int64, smallBlockSize int64, ctx *ECContext, additionalDirs []string) (generatedShardIds []uint32, err error) {
|
||||
@@ -150,22 +161,35 @@ func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize
|
||||
shardHasData := make([]bool, ctx.Total())
|
||||
shardPaths := make([]string, ctx.Total()) // non-empty for present shards
|
||||
inputFiles := make([]*os.File, ctx.Total())
|
||||
var ghostPaths []string
|
||||
expectedShardSize := int64(-1)
|
||||
presentCount := 0
|
||||
for shardId := 0; shardId < ctx.Total(); shardId++ {
|
||||
ext := ctx.ToExt(shardId)
|
||||
shardPath := findShardFile(baseFileName, ext, additionalDirs)
|
||||
if shardPath != "" {
|
||||
shardHasData[shardId] = true
|
||||
shardPaths[shardId] = shardPath
|
||||
inputFiles[shardId], err = os.OpenFile(shardPath, os.O_RDONLY, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer inputFiles[shardId].Close()
|
||||
presentCount++
|
||||
} else {
|
||||
shardPath, ghosts := findShardFile(baseFileName, ext, additionalDirs)
|
||||
ghostPaths = append(ghostPaths, ghosts...)
|
||||
if shardPath == "" {
|
||||
generatedShardIds = append(generatedShardIds, uint32(shardId))
|
||||
continue
|
||||
}
|
||||
fi, statErr := os.Stat(shardPath)
|
||||
if statErr != nil {
|
||||
return nil, fmt.Errorf("stat shard %s: %w", shardPath, statErr)
|
||||
}
|
||||
if expectedShardSize < 0 {
|
||||
expectedShardSize = fi.Size()
|
||||
} else if fi.Size() != expectedShardSize {
|
||||
return nil, fmt.Errorf("ec shard size mismatch: %s is %d bytes, expected %d (refusing to rebuild from inconsistent shards)",
|
||||
shardPath, fi.Size(), expectedShardSize)
|
||||
}
|
||||
shardHasData[shardId] = true
|
||||
shardPaths[shardId] = shardPath
|
||||
inputFiles[shardId], err = os.OpenFile(shardPath, os.O_RDONLY, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer inputFiles[shardId].Close()
|
||||
presentCount++
|
||||
}
|
||||
|
||||
// Pre-check: bail out before creating any output files.
|
||||
@@ -174,8 +198,8 @@ func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize
|
||||
baseFileName, presentCount, ctx.DataShards, generatedShardIds)
|
||||
}
|
||||
|
||||
glog.V(0).Infof("rebuilding %s: %d shards present, %d missing %v, config %s",
|
||||
baseFileName, presentCount, len(generatedShardIds), generatedShardIds, ctx.String())
|
||||
glog.V(0).Infof("rebuilding %s: %d shards present (size %d), %d missing %v, %d ghost residue %v, config %s",
|
||||
baseFileName, presentCount, expectedShardSize, len(generatedShardIds), generatedShardIds, len(ghostPaths), ghostPaths, ctx.String())
|
||||
|
||||
// Pass 2: create output files for missing shards now that we know
|
||||
// reconstruction is possible.
|
||||
@@ -192,10 +216,32 @@ func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize
|
||||
defer outputFiles[shardId].Close()
|
||||
}
|
||||
|
||||
err = rebuildEcFiles(shardHasData, inputFiles, outputFiles, ctx)
|
||||
err = rebuildEcFiles(shardHasData, inputFiles, outputFiles, ctx, expectedShardSize)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rebuildEcFiles: %w", err)
|
||||
}
|
||||
|
||||
// Reconstruction succeeded — remove any 0-byte residue from prior aborted
|
||||
// rebuilds so they don't shadow real shards on the next pass and don't get
|
||||
// mounted as empty stripes. Skip residue paths that the output-file step
|
||||
// has already overwritten with real data, since those entries in
|
||||
// ghostPaths refer to the just-rebuilt shard's location.
|
||||
freshOutputs := make(map[string]struct{}, len(outputFiles))
|
||||
for _, f := range outputFiles {
|
||||
if f != nil {
|
||||
freshOutputs[f.Name()] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, ghost := range ghostPaths {
|
||||
if _, overwritten := freshOutputs[ghost]; overwritten {
|
||||
continue
|
||||
}
|
||||
if removeErr := os.Remove(ghost); removeErr != nil && !os.IsNotExist(removeErr) {
|
||||
glog.Warningf("failed to remove 0-byte ec shard residue %s: %v", ghost, removeErr)
|
||||
} else {
|
||||
glog.V(0).Infof("removed 0-byte ec shard residue %s", ghost)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -320,7 +366,7 @@ func encodeDatFile(remainingSize int64, baseFileName string, bufferSize int, lar
|
||||
return nil
|
||||
}
|
||||
|
||||
func rebuildEcFiles(shardHasData []bool, inputFiles []*os.File, outputFiles []*os.File, ctx *ECContext) error {
|
||||
func rebuildEcFiles(shardHasData []bool, inputFiles []*os.File, outputFiles []*os.File, ctx *ECContext, expectedShardSize int64) error {
|
||||
|
||||
enc, err := ctx.CreateEncoder()
|
||||
if err != nil {
|
||||
@@ -334,23 +380,22 @@ func rebuildEcFiles(shardHasData []bool, inputFiles []*os.File, outputFiles []*o
|
||||
}
|
||||
}
|
||||
|
||||
var startOffset int64
|
||||
var inputBufferDataSize int
|
||||
for {
|
||||
for startOffset := int64(0); startOffset < expectedShardSize; {
|
||||
chunkSize := int64(ErasureCodingSmallBlockSize)
|
||||
if remaining := expectedShardSize - startOffset; remaining < chunkSize {
|
||||
chunkSize = remaining
|
||||
}
|
||||
|
||||
// read the input data from files
|
||||
for i := 0; i < ctx.Total(); i++ {
|
||||
if shardHasData[i] {
|
||||
n, _ := inputFiles[i].ReadAt(buffers[i], startOffset)
|
||||
if n == 0 {
|
||||
return nil
|
||||
}
|
||||
if inputBufferDataSize == 0 {
|
||||
inputBufferDataSize = n
|
||||
}
|
||||
if inputBufferDataSize != n {
|
||||
return fmt.Errorf("ec shard size expected %d actual %d", inputBufferDataSize, n)
|
||||
buf := buffers[i][:chunkSize]
|
||||
n, readErr := inputFiles[i].ReadAt(buf, startOffset)
|
||||
if int64(n) != chunkSize {
|
||||
return fmt.Errorf("short read on shard %s at offset %d: got %d, want %d (err=%v)",
|
||||
inputFiles[i].Name(), startOffset, n, chunkSize, readErr)
|
||||
}
|
||||
buffers[i] = buf
|
||||
} else {
|
||||
buffers[i] = nil
|
||||
}
|
||||
@@ -365,15 +410,16 @@ func rebuildEcFiles(shardHasData []bool, inputFiles []*os.File, outputFiles []*o
|
||||
// write the data to output files
|
||||
for i := 0; i < ctx.Total(); i++ {
|
||||
if !shardHasData[i] {
|
||||
n, _ := outputFiles[i].WriteAt(buffers[i][:inputBufferDataSize], startOffset)
|
||||
if inputBufferDataSize != n {
|
||||
return fmt.Errorf("fail to write to %s", outputFiles[i].Name())
|
||||
n, writeErr := outputFiles[i].WriteAt(buffers[i][:chunkSize], startOffset)
|
||||
if int64(n) != chunkSize {
|
||||
return fmt.Errorf("short write to %s at offset %d: got %d, want %d (err=%v)",
|
||||
outputFiles[i].Name(), startOffset, n, chunkSize, writeErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
startOffset += int64(inputBufferDataSize)
|
||||
startOffset += chunkSize
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func readNeedleMap(baseFileName string) (*needle_map.MemDb, error) {
|
||||
|
||||
Reference in new issue
Block a user