EC decode: read shards with the encode-time block layout (#10385)

* erasure_coding: WriteDatFile takes the encode-time dat size for the shard block layout

* volume server: derive EC decode layout from the encode-time dat size, not the live extent

* erasure_coding: test decode after tail deletions shrink the live extent below a large-block row

* seaweed-volume: write_dat_file_from_shards takes the encode-time dat size for the shard block layout

* seaweed-volume: derive EC decode layout from the encode-time dat size, not the live extent

* seaweed-volume: test decode after tail deletions shrink the live extent below a large-block row

* erasure_coding: reject decoding with no data shards

* worker: record the encode-time dat size in the .vif

* erasure_coding: fall back to the shard-derived layout only when the encode-time dat size is missing

* erasure_coding: reject an ambiguous shard-derived block layout

* seaweed-volume: fall back to the shard-derived layout only when the encode-time dat size is missing

* seaweed-volume: reject an ambiguous shard-derived block layout
This commit is contained in:
Chris Lu authored and GitHub committed 2026-07-21 08:59:14 -07:00
1 parent cdb60069a6
commit 5a54beac80
9 files changed
+516 -31

No files matched your search

@@ -75,6 +75,8 @@ pub fn write_ec_files(
&mut builders,
data_shards,
parity_shards,
ERASURE_CODING_LARGE_BLOCK_SIZE,
ERASURE_CODING_SMALL_BLOCK_SIZE,
)?;
// Close all shards
@@ -582,7 +584,8 @@ fn read_from_data_shards(
/// Uses a two-phase approach matching Go's ec_encoder.go:
/// 1. Process as many large blocks (1GB) as possible
/// 2. Process remaining data with small blocks (1MB)
fn encode_dat_file(
#[allow(clippy::too_many_arguments)]
pub(crate) fn encode_dat_file(
dat_file: &File,
dat_size: i64,
rs: &ReedSolomon,
@@ -590,12 +593,13 @@ fn encode_dat_file(
builders: &mut [ShardChecksumBuilder],
data_shards: usize,
parity_shards: usize,
large_block_size: usize,
small_block_size: usize,
) -> io::Result<()> {
let mut remaining = dat_size;
let mut offset: u64 = 0;
// Phase 1: Process large blocks (1GB each) while enough data remains
let large_block_size = ERASURE_CODING_LARGE_BLOCK_SIZE;
let large_row_size = large_block_size * data_shards;
while remaining >= large_row_size as i64 {
@@ -614,7 +618,6 @@ fn encode_dat_file(
}
// Phase 2: Process remaining data with small blocks (1MB each)
let small_block_size = ERASURE_CODING_SMALL_BLOCK_SIZE;
let small_row_size = small_block_size * data_shards;
while remaining > 0 {