mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-05 19:47:47 +01:00
EC decode: read shards with the encode-time block layout (#10385)
* erasure_coding: WriteDatFile takes the encode-time dat size for the shard block layout * volume server: derive EC decode layout from the encode-time dat size, not the live extent * erasure_coding: test decode after tail deletions shrink the live extent below a large-block row * seaweed-volume: write_dat_file_from_shards takes the encode-time dat size for the shard block layout * seaweed-volume: derive EC decode layout from the encode-time dat size, not the live extent * seaweed-volume: test decode after tail deletions shrink the live extent below a large-block row * erasure_coding: reject decoding with no data shards * worker: record the encode-time dat size in the .vif * erasure_coding: fall back to the shard-derived layout only when the encode-time dat size is missing * erasure_coding: reject an ambiguous shard-derived block layout * seaweed-volume: fall back to the shard-derived layout only when the encode-time dat size is missing * seaweed-volume: reject an ambiguous shard-derived block layout
This commit is contained in:
1 parent
cdb60069a6
commit
5a54beac80
9 files changed
+516
-31
No files matched your search
@@ -75,6 +75,8 @@ pub fn write_ec_files(
|
||||
&mut builders,
|
||||
data_shards,
|
||||
parity_shards,
|
||||
ERASURE_CODING_LARGE_BLOCK_SIZE,
|
||||
ERASURE_CODING_SMALL_BLOCK_SIZE,
|
||||
)?;
|
||||
|
||||
// Close all shards
|
||||
@@ -582,7 +584,8 @@ fn read_from_data_shards(
|
||||
/// Uses a two-phase approach matching Go's ec_encoder.go:
|
||||
/// 1. Process as many large blocks (1GB) as possible
|
||||
/// 2. Process remaining data with small blocks (1MB)
|
||||
fn encode_dat_file(
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn encode_dat_file(
|
||||
dat_file: &File,
|
||||
dat_size: i64,
|
||||
rs: &ReedSolomon,
|
||||
@@ -590,12 +593,13 @@ fn encode_dat_file(
|
||||
builders: &mut [ShardChecksumBuilder],
|
||||
data_shards: usize,
|
||||
parity_shards: usize,
|
||||
large_block_size: usize,
|
||||
small_block_size: usize,
|
||||
) -> io::Result<()> {
|
||||
let mut remaining = dat_size;
|
||||
let mut offset: u64 = 0;
|
||||
|
||||
// Phase 1: Process large blocks (1GB each) while enough data remains
|
||||
let large_block_size = ERASURE_CODING_LARGE_BLOCK_SIZE;
|
||||
let large_row_size = large_block_size * data_shards;
|
||||
|
||||
while remaining >= large_row_size as i64 {
|
||||
@@ -614,7 +618,6 @@ fn encode_dat_file(
|
||||
}
|
||||
|
||||
// Phase 2: Process remaining data with small blocks (1MB each)
|
||||
let small_block_size = ERASURE_CODING_SMALL_BLOCK_SIZE;
|
||||
let small_row_size = small_block_size * data_shards;
|
||||
|
||||
while remaining > 0 {
|
||||
|
||||
Reference in new issue
Block a user