From 59228cf809fca5252400fc0b2c4702137b7b4d84 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 7 May 2026 23:29:03 -0700 Subject: [PATCH] perf(ec): pre-allocate missing-shard buffers across rebuild chunks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reconstruct previously got nil for each missing shard and allocated a fresh 1MB buffer per chunk — thousands of allocations on a 1GB volume. klauspost/reedsolomon reuses cap when len==0, so handing it a 0-len slice with the existing capacity avoids the allocation while still signaling "missing". --- weed/storage/erasure_coding/ec_encoder.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/weed/storage/erasure_coding/ec_encoder.go b/weed/storage/erasure_coding/ec_encoder.go index b1e6239df..51ea908b6 100644 --- a/weed/storage/erasure_coding/ec_encoder.go +++ b/weed/storage/erasure_coding/ec_encoder.go @@ -370,11 +370,13 @@ func rebuildEcFiles(shardHasData []bool, inputFiles []*os.File, outputFiles []*o return fmt.Errorf("failed to create encoder: %w", err) } + // Pre-allocate buffers for every shard, including the missing ones we + // need to reconstruct. reedsolomon.Reconstruct reuses cap when len==0 + // (cap(shard) >= shardSize → shard[0:shardSize]), so handing it a 0-len + // slice with the right cap avoids a fresh allocation per chunk. buffers := make([][]byte, ctx.Total()) for i := range buffers { - if shardHasData[i] { - buffers[i] = make([]byte, ErasureCodingSmallBlockSize) - } + buffers[i] = make([]byte, ErasureCodingSmallBlockSize) } for startOffset := int64(0); startOffset < expectedShardSize; { @@ -394,7 +396,9 @@ func rebuildEcFiles(shardHasData []bool, inputFiles []*os.File, outputFiles []*o } buffers[i] = buf } else { - buffers[i] = nil + // 0-len, full-cap: signals "missing" while letting Reconstruct + // reslice into our existing storage. + buffers[i] = buffers[i][:0] } }