seaweedfs/seaweed-volume/src/storage/erasure_coding/ec_volume.rs

//! EcVolume: an erasure-coded volume with up to MAX_SHARD_COUNT shards.
//!
//! Each EcVolume has a sorted index (.ecx) and a deletion journal (.ecj).
//! Shards (.ec00-.ec13) may be distributed across multiple servers.

use std::collections::{HashMap, HashSet};
use std::fs::{self, File, OpenOptions};
use std::io::{self, BufWriter, Write};
use std::sync::RwLock;
use std::time::{Instant, SystemTime, UNIX_EPOCH};

use crate::pb::master_pb;
use crate::storage::erasure_coding::ec_locate;
use crate::storage::erasure_coding::ec_shard::*;
use crate::storage::erasure_coding::ecj_registry::EcjHold;
use crate::storage::io::read_exact_at;
use crate::storage::io_error::IoErrorTracker;
use crate::storage::needle::needle::{Needle, NeedleError, get_actual_size};
use crate::storage::types::*;
use crate::storage::volume_open::open_volume_file;

/// The shard-location cache: where each shard lives, when that was last learned
/// from the master, and whether a read has since disproved it. One struct behind
/// one lock, because the freshness heuristic judges all three together (the map
/// and its time from the reader's own snapshot, see
/// `EcVolume::claim_shard_locations_refresh`) — a map published ahead of its
/// refresh time reads as a fresh map stamped with the previous lookup, and a
/// half-populated map must never look fresh at all.
#[derive(Default)]
pub(crate) struct ShardLocationCache {
    /// Maps shard ID -> list of server addresses where that shard exists.
    /// Used for distributed EC reads across the cluster.
    locations: HashMap<ShardId, Vec<String>>,
    /// Wall-clock timestamp of the most recent successful `LookupEcVolume`
    /// refresh of `locations`. `None` until the first refresh. Drives the
    /// staleness heuristic in `cached_lookup_ec_shard_locations` (mirrors Go's
    /// `ShardLocationsRefreshTime`).
    refreshed_at: Option<Instant>,
    /// Marks the map for a prompt re-check: a read that failed against a cached
    /// location has disproved what the map claims, and the normal freshness
    /// window is far too long to serve from a map known to be wrong. Mirrors the
    /// invalidation Go's `forgetShardId` performs. A field under the map's lock
    /// rather than an atomic so the refresh can consume the mark in one critical
    /// section, and a mark raised meanwhile survives for the next refresh.
    stale: bool,
}

/// Bytes read per positional read when seeding `deleted_needles` from `.ecj`.
/// A multiple of `NEEDLE_ID_SIZE`; 1 MiB is 131072 entries per syscall.
const ECJ_LOAD_CHUNK_BYTES: usize = 1 << 20;

/// Process-wide epoch bumped whenever a failed .ecj append truncates its
/// uncommitted tail. A decode that read the journal without the store lock
/// compares a sample taken before the read with one taken under the
/// quiescing lock: equal means no rollback ran in between, so every id it
/// folded in was committed and only incremental catch-up remains — the
/// full rescan is needed only when the epoch moved.
pub(crate) static ECJ_ROLLBACK_EPOCH: std::sync::atomic::AtomicU64 =
    std::sync::atomic::AtomicU64::new(0);

/// A `.ecj` smaller than this is never rewritten, however redundant. Below a
/// megabyte the duplication costs nothing and the rewrite is pure churn.
const ECJ_COMPACT_MIN_BYTES: i64 = 1 << 20;

/// Rewrite only when the journal is at least this many times larger than the
/// set it encodes. A journal holds one entry per delete, so a healthy one is
/// close to 1x; four times the set means most of the file is repeats.
const ECJ_COMPACT_RATIO: i64 = 4;

/// Staging file for a compacted journal, next to the `.ecj` it replaces.
/// Listed with the other EC index files wherever those are removed, so a tmp
/// left by a crash between write and rename does not outlive its volume.
pub(crate) const ECJ_COMPACT_TMP_EXT: &str = ".ecj.compact.tmp";

/// Open `path` as a deletion journal append handle, creating it if absent.
/// Every handle `journal_delete` appends through comes from here, so a handle
/// reopened after compaction behaves exactly like the one opened at mount.
fn open_ecj_append(path: &str) -> io::Result<File> {
    open_volume_file(
        OpenOptions::new()
            .read(true)
            .write(true)
            .create(true)
            .append(true),
        path,
    )
}

/// The filesystem steps of a mount's journal compaction: the reads that load
/// the set it compacts from, and the steps that publish the compacted file.
/// Production uses [`EcjFsOps::REAL`]; tests substitute their own steps to
/// cover the failure paths and races through the real mount.
#[derive(Clone, Copy)]
struct EcjFsOps {
    read_at: fn(&File, &mut [u8], u64) -> io::Result<()>,
    rename: fn(&str, &str) -> io::Result<()>,
    fsync_dir: fn(&str) -> io::Result<()>,
    reopen: fn(&str) -> io::Result<File>,
}

impl EcjFsOps {
    const REAL: EcjFsOps = EcjFsOps {
        read_at: read_exact_at,
        rename: |from, to| fs::rename(from, to),
        fsync_dir: fsync_ecj_dir,
        reopen: open_ecj_append,
    };
}

/// Fsync the directory holding `path`, failing when it cannot be opened.
/// The crate's `fsync_dir` treats an unopenable directory as success, which is
/// best-effort syncing; here the mount carries on after the rename and takes
/// deletes against the replacement, so an unsynced entry could lose them to a
/// power loss. A rename needs only write and search permission on the
/// directory, so it can succeed where opening the directory fails. Matches
/// Go's `util.FsyncDir`, which also skips the sync on Windows.
fn fsync_ecj_dir(path: &str) -> io::Result<()> {
    #[cfg(windows)]
    {
        let _ = path;
        Ok(())
    }
    #[cfg(not(windows))]
    {
        let dir = match std::path::Path::new(path).parent() {
            Some(p) if !p.as_os_str().is_empty() => p,
            _ => std::path::Path::new("."),
        };
        File::open(dir)?.sync_all()
    }
}

/// Why publishing a compacted journal failed, split by whether the volume
/// still has a working journal handle. Decided where the failure happens, not
/// inferred afterwards from `ecj_file`.
#[derive(Debug)]
enum EcjPublishError {
    /// Nothing was published: the original journal is in place and the handle
    /// to it is restored. The mount carries on with the uncompacted file.
    Abandoned(io::Error),
    /// The volume has no usable journal handle, so it must not mount: deletes
    /// would fail, or land in an inode no longer at the journal's path.
    HandleLost(io::Error),
}

/// Adds every whole needle id in bytes `from..len` of `ecj_file` to `ids`,
/// reading `ECJ_LOAD_CHUNK_BYTES` at a time; a trailing partial record is
/// ignored. `from` is a record boundary.
pub(crate) fn read_ecj_ids(
    ecj_file: &File,
    from: u64,
    len: u64,
    ids: &mut HashSet<NeedleId>,
) -> io::Result<()> {
    read_ecj_ids_with(read_exact_at, ecj_file, from, len, ids)
}

/// [`read_ecj_ids`] with the positional read supplied by the caller, so the
/// mount can load through its injectable [`EcjFsOps`] steps.
fn read_ecj_ids_with(
    read_at: fn(&File, &mut [u8], u64) -> io::Result<()>,
    ecj_file: &File,
    from: u64,
    len: u64,
    ids: &mut HashSet<NeedleId>,
) -> io::Result<()> {
    let mut buf =
        vec![0u8; std::cmp::min(ECJ_LOAD_CHUNK_BYTES as u64, len.saturating_sub(from)) as usize];
    let mut off: u64 = from;
    while off + NEEDLE_ID_SIZE as u64 <= len {
        let mut want = std::cmp::min(ECJ_LOAD_CHUNK_BYTES as u64, len - off) as usize;
        want -= want % NEEDLE_ID_SIZE;
        // Positional read: the loader's handle is shared with journal appends.
        read_at(ecj_file, &mut buf[..want], off)?;
        for entry in buf[..want].as_chunks::<NEEDLE_ID_SIZE>().0 {
            ids.insert(NeedleId::from_bytes(entry));
        }
        off += want as u64;
    }
    Ok(())
}

/// An erasure-coded volume managing its local shards and index.
pub struct EcVolume {
    pub volume_id: VolumeId,
    pub collection: String,
    pub dir: String,
    pub dir_idx: String,
    pub version: Version,
    pub shards: Vec<Option<EcVolumeShard>>, // indexed by ShardId (0..MAX_SHARD_COUNT)
    pub dat_file_size: i64,
    pub data_shards: u32,
    pub parity_shards: u32,
    /// Uniform block layout: each shard is one contiguous block of this many
    /// bytes. 0 = legacy 1GiB/1MiB two-tier layout. Loaded from .vif.
    pub block_size: i64,
    ecx_file: Option<File>,
    ecx_file_size: i64,
    ecj_file: Option<File>,
    /// This volume's registration as a holder of its `.ecj` path. Held for as
    /// long as `ecj_file` may be open; see `ecj_registry`.
    ecj_hold: Option<EcjHold>,
    /// On-disk size of the .ecj deletion journal: the rollback point for a
    /// failed append, and the size mount-time compaction compares against the
    /// id set and re-checks on disk before replacing the file. The runtime
    /// delete count comes from `deleted_needles.len()`, not from this.
    ecj_file_size: i64,
    /// In-memory set of needle ids that have been deleted since the volume
    /// was encoded. .ecx is immutable at runtime — it only stores the
    /// sorted (id, offset, size) index written at encode time — and runtime
    /// deletes are journaled to .ecj + tracked here. Reads consult this
    /// set to mask out deleted needles on top of the sealed .ecx lookup.
    /// Seeded from .ecj in `new()` and updated by `journal_delete`.
    deleted_needles: RwLock<HashSet<NeedleId>>,
    pub disk_type: DiskType,
    /// Directory where .ecx/.ecj were actually found (may differ from dir_idx after fallback).
    ecx_actual_dir: String,
    /// Where each shard lives and how far that is to be trusted, all under one
    /// `RwLock` so the read path can refresh it (under master lookup) without
    /// holding the Store write lock — mirrors Go's `ShardLocationsLock
    /// sync.RWMutex` in `weed/storage/erasure_coding/ec_volume.go`.
    shard_locations: RwLock<ShardLocationCache>,
    /// EC volume expiration time (unix epoch seconds), set during EC encode from TTL.
    pub expire_at_sec: u64,
    /// Encode-run identity (unix nanos) loaded from the .vif EcShardConfig. A read
    /// served from a shard of a different encode run is rejected (server- and
    /// client-side); 0 for a pre-feature volume, which is treated leniently.
    pub encode_ts_ns: i64,
    /// Active-generation EC bitrot checksum sidecar (`<base>.ecsum`), loaded and
    /// validated at mount. `None` unless `bitrot_status == On`.
    pub(crate) bitrot: Option<crate::pb::volume_server_pb::EcBitrotProtection>,
    /// Resolved protection state of the loaded sidecar. Cached alongside `bitrot`
    /// so `bitrot_protection()` can return the `Off`/`Invalid` distinction without
    /// re-reading, mirroring Go's `EcVolume.bitrotStatus`.
    pub(crate) bitrot_status: crate::storage::erasure_coding::ec_bitrot::BitrotStatus,
    /// Directory the active sidecar was actually resolved from. The search in
    /// `load_bitrot_for_generation` spans sibling disks, and a `.ecsum` records
    /// no encode identity, so provenance is the only signal that a loaded
    /// manifest may describe a different encode run. Same purpose as
    /// `ecx_actual_dir`. Empty when no sidecar was found. Always committed in
    /// the same step as `bitrot`/`bitrot_status`, so a reload whose `Err` gets
    /// swallowed by `reload_bitrot_sidecar` cannot leave this describing a
    /// sidecar other than the one those two fields actually hold.
    pub(crate) bitrot_source_dir: String,

    /// Consecutive storage-media errors and the quarantine they lead to,
    /// for EC volume health monitoring.
    io_errors: IoErrorTracker,
}

/// Locate the `.vif` for a (collection, vid) by preferring the data dir
/// and falling back to the idx dir when it lives there instead. The
/// fallback covers the cross-disk reconcile path: when a volume's
/// shards live on one disk but its `.ecx` / `.ecj` / `.vif` live on a
/// sibling disk (seaweedfs/seaweedfs#9212 / #9244), we want to read the
/// real `.vif` from the sibling rather than write a stub on the shard
/// disk and lose the EC config + dat file size.
fn locate_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: VolumeId) -> String {
    let data_vif = format!(
        "{}.vif",
        crate::storage::volume::volume_file_name(dir, collection, volume_id),
    );
    if dir_idx != dir && !std::path::Path::new(&data_vif).exists() {
        let idx_vif = format!(
            "{}.vif",
            crate::storage::volume::volume_file_name(dir_idx, collection, volume_id),
        );
        if std::path::Path::new(&idx_vif).exists() {
            return idx_vif;
        }
    }
    data_vif
}

/// Load the volume's `.vif` (data dir first, then idx dir, then the active
/// generation's versioned sidecar — see [`locate_vif_path`]). Absent
/// everywhere is legal — legacy volumes predate the sidecar — and returns
/// `None`; so does a zero-byte stub (an ec.decode copy from a source
/// without one), mirroring Go's MaybeLoadVolumeInfo. A
/// present-but-unreadable or malformed `.vif` is an ERROR: silently
/// defaulting would mount a uniform-layout volume with legacy offset math
/// and serve wrong bytes with a straight face.
pub fn load_vif_info(
    dir: &str,
    dir_idx: &str,
    collection: &str,
    volume_id: VolumeId,
) -> io::Result<Option<crate::storage::volume::VifVolumeInfo>> {
    let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
    match std::fs::read_to_string(&vif_path) {
        Ok(content) if content.trim().is_empty() => Ok(None),
        Ok(content) => serde_json::from_str(&content).map(Some).map_err(|e| {
            io::Error::new(
                io::ErrorKind::InvalidData,
                format!("parse {}: {}", vif_path, e),
            )
        }),
        Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None),
        Err(e) => Err(io::Error::new(
            e.kind(),
            format!("read {}: {}", vif_path, e),
        )),
    }
}

/// Read the EC data/parity shard counts and shard block size from `.vif`,
/// defaulting to the build's standard ratio and the legacy block layout when
/// no `.vif` is present. A present-but-unreadable or malformed `.vif`
/// fails instead of defaulting (see [`load_vif_info`]).
/// Looks at the data dir first, then the idx dir — see [`locate_vif_path`].
pub fn read_ec_shard_config(
    dir: &str,
    dir_idx: &str,
    collection: &str,
    volume_id: VolumeId,
) -> io::Result<(u32, u32, i64)> {
    let vif = load_vif_info(dir, dir_idx, collection, volume_id)?;
    ec_shard_config_from(vif.as_ref(), &[dir, dir_idx], collection, volume_id)
}

/// Load the volume's `.vif` from the selected location or, failing that, from
/// any sibling disk — returning the directory it came from so the caller can
/// resolve the rest of the volume's metadata against the same place. A rebuild
/// picks one disk to write into, but a multi-disk server may hold that
/// volume's metadata on another.
pub fn load_vif_info_across_dirs(
    dir: &str,
    dir_idx: &str,
    other_dirs: &[String],
    collection: &str,
    volume_id: VolumeId,
) -> io::Result<Option<(crate::storage::volume::VifVolumeInfo, String)>> {
    if let Some(vif) = load_vif_info(dir, dir_idx, collection, volume_id)? {
        // Report where it actually came from. load_vif_info probes the data
        // directory then the index directory, so naming `dir` unconditionally
        // pointed callers at the wrong disk whenever the vif lived with the
        // index — and a caller resolving the rest of the volume's metadata
        // against that answer would look in a directory holding none of it.
        let data_vif = format!(
            "{}.vif",
            crate::storage::volume::volume_file_name(dir, collection, volume_id)
        );
        let found_in = if std::path::Path::new(&data_vif).exists() {
            dir.to_string()
        } else {
            dir_idx.to_string()
        };
        return Ok(Some((vif, found_in)));
    }
    for other in other_dirs {
        if let Some(vif) = load_vif_info(other, other, collection, volume_id)? {
            return Ok(Some((vif, other.clone())));
        }
    }
    Ok(None)
}

/// [`read_ec_shard_config`] for a caller that may find the volume's metadata on
/// any of the server's disks. Without this a rebuild that lands on a disk
/// holding only shards reads the default 10+4 and the legacy block layout,
/// then reconstructs a custom-ratio or uniform volume through the wrong matrix
/// and de-striping geometry.
pub fn read_ec_shard_config_across_dirs(
    dir: &str,
    dir_idx: &str,
    other_dirs: &[String],
    collection: &str,
    volume_id: VolumeId,
) -> io::Result<(u32, u32, i64)> {
    // Every directory this volume's metadata could be in. A split -dir/-dir.idx
    // location keeps .vif and .ecsum with the INDEX, and callers exclude their
    // own index directory from other_dirs on the understanding that it is
    // passed here separately — so it is chained explicitly, exactly as the .vif
    // lookup and Go's findBitrotSidecar both do.
    let mut candidates: Vec<&str> = Vec::with_capacity(2 + other_dirs.len());
    candidates.push(dir);
    if !dir_idx.is_empty() && dir_idx != dir {
        candidates.push(dir_idx);
    }
    candidates.extend(other_dirs.iter().map(|s| s.as_str()));

    // A vif that carries no ecShardConfig answers nothing about the layout, so
    // it must NOT short-circuit the sidecar search: a legacy config-free vif
    // and the generation-0 sidecar can sit in different directories, and
    // stopping at the vif resolved a 12+4 uniform volume as 10+4 legacy. Pass
    // the whole candidate list so the fallback covers the same ground the vif
    // lookup did.
    let vif = load_vif_info_across_dirs(dir, dir_idx, other_dirs, collection, volume_id)?;
    ec_shard_config_from(
        vif.as_ref().map(|(v, _)| v),
        &candidates,
        collection,
        volume_id,
    )
}

/// Resolve (data_shards, parity_shards, block_size) from an already-loaded
/// `.vif`. With no vif — or one carrying no EC config — the bitrot sidecar
/// records the same config at encode time and answers the layout question the
/// vif cannot: defaulting a uniform-layout volume to the legacy block sizes
/// maps every read to the wrong shard offset. `weed fix -ecx` reads the
/// sidecar for the same reason. Absent both, the build's standard ratio and
/// the legacy layout.
pub fn ec_shard_config_from(
    vif: Option<&crate::storage::volume::VifVolumeInfo>,
    dirs: &[&str],
    collection: &str,
    volume_id: VolumeId,
) -> io::Result<(u32, u32, i64)> {
    let default_ds = crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT as u32;
    let default_ps = crate::storage::erasure_coding::ec_shard::PARITY_SHARDS_COUNT as u32;
    // Sum as u64: counts near the u32 ceiling would wrap and pass the bound.
    let usable =
        |ds: u32, ps: u32| ds > 0 && ps > 0 && (ds as u64 + ps as u64) <= MAX_SHARD_COUNT as u64;

    if let Some(ec) = vif.and_then(|v| v.ec_shard_config.as_ref()) {
        // A config that is PRESENT but records an impossible ratio is not a
        // volume to fall back on: dropping through to the sidecar or the
        // defaults would read uniform shards with the legacy offset math and
        // answer with the wrong bytes. Only an ENTIRELY absent config means
        // "this predates the record".
        if !usable(ec.data_shards, ec.parity_shards) {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!(
                    "vif for volume {} records invalid shard counts {}+{}",
                    volume_id.0, ec.data_shards, ec.parity_shards
                ),
            ));
        }
        // A recorded block size that no encoder could have produced maps
        // every read to the wrong shard offset. Refuse the mount rather
        // than serve those bytes or silently pick a layout.
        validate_block_size(ec.block_size)?;
        return Ok((ec.data_shards, ec.parity_shards, ec.block_size));
    }
    // With no usable vif config the sidecar is the ONLY record of this volume's
    // layout, so the three answers stay distinct: absent EVERYWHERE means the
    // volume may genuinely predate the sidecar and legacy is the right guess;
    // present-but-unusable means the record is corrupt, and answering reads
    // from a guessed layout returns wrong bytes rather than none.
    //
    // Every candidate directory is searched, not just the first: a split
    // -dir/-dir.idx location keeps the sidecar with the INDEX, and a multi-disk
    // server may keep it on a sibling. Stopping at the data directory resolved
    // a 12+4 uniform volume as 10+4 legacy.
    let mut path = String::new();
    for candidate in dirs.iter().filter(|d| !d.is_empty()) {
        let base = crate::storage::volume::volume_file_name(candidate, collection, volume_id);
        let probe = crate::storage::erasure_coding::ec_bitrot::bitrot_sidecar_path(&base, 0);
        match std::fs::metadata(&probe) {
            Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
            Err(e) => {
                return Err(io::Error::new(e.kind(), format!("stat {}: {}", probe, e)));
            }
            Ok(_) => {
                path = probe;
                break;
            }
        }
    }
    if path.is_empty() {
        return Ok((default_ds, default_ps, 0));
    }
    let prot = crate::storage::erasure_coding::ec_bitrot::load_bitrot_sidecar(&path)
        .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("read {}: {}", path, e)))?;
    // The un-suffixed sidecar describes generation 0 and nothing else.
    if prot.generation != 0 {
        return Err(io::Error::new(
            io::ErrorKind::InvalidData,
            format!(
                "{} records generation {}, not generation 0",
                path, prot.generation
            ),
        ));
    }
    let ec = prot.ec_shard_config.as_ref().ok_or_else(|| {
        io::Error::new(
            io::ErrorKind::InvalidData,
            format!("{} records no EC config", path),
        )
    })?;
    if !usable(ec.data_shards, ec.parity_shards) {
        return Err(io::Error::new(
            io::ErrorKind::InvalidData,
            format!(
                "{} records invalid shard counts {}+{}",
                path, ec.data_shards, ec.parity_shards
            ),
        ));
    }
    validate_block_size(ec.block_size)
        .map_err(|e| io::Error::new(e.kind(), format!("{}: {}", path, e)))?;
    Ok((ec.data_shards, ec.parity_shards, ec.block_size))
}

/// Reports whether a `.vif`-recorded shard block size is one an encoder could
/// have produced. 0 means the legacy two-tier layout, which is always valid;
/// anything positive must be a whole number of small blocks, because that is
/// what `uniform_block_size` rounds to. Mirrors Go's `ValidateBlockSize`.
pub fn validate_block_size(block_size: i64) -> io::Result<()> {
    if block_size == 0 {
        return Ok(());
    }
    if block_size < 0
        || block_size
            % crate::storage::erasure_coding::ec_shard::ERASURE_CODING_SMALL_BLOCK_SIZE as i64
            != 0
    {
        return Err(io::Error::new(
            io::ErrorKind::InvalidData,
            format!(
                "invalid shard block size {}: expected 0 (legacy) or a multiple of {}",
                block_size,
                crate::storage::erasure_coding::ec_shard::ERASURE_CODING_SMALL_BLOCK_SIZE
            ),
        ));
    }
    Ok(())
}

/// Size of the `.ecx` at `path`, or `None` when it is absent or a directory.
/// Mirrors Go's `statEcxSize`.
pub(crate) fn ecx_file_size(path: &str) -> Option<u64> {
    match std::fs::metadata(path) {
        Ok(meta) if !meta.is_dir() => Some(meta.len()),
        _ => None,
    }
}

/// Whether `path` is an `.ecx` that can steer placement, ownership or a copy:
/// a regular file with content. A 0-byte `.ecx` is what a failed EC distribute
/// copy leaves behind, and Go treats it as absent at every such decision
/// (`HasEcxFileOnDisk`, `findEcxIdxDirForVolume`, `indexEcxOwners`) so the scan
/// moves on to a sibling disk that may hold a valid index.
pub(crate) fn is_usable_ecx_file(path: &str) -> bool {
    ecx_file_size(path).is_some_and(|size| size > 0)
}

/// Stream sorted deleted ids to `tmp_path` and fsync it. Extracted so failure
/// paths (ENOSPC, read-only dir) are unit-testable without mounting a volume.
fn write_compacted_ecj_tmp(tmp_path: &str, ids: &[NeedleId]) -> io::Result<()> {
    // Stream the ids out rather than materialising the whole encoded
    // journal: the set and the sorted vector are already resident, and a
    // third full-size buffer is a needless contiguous allocation at mount.
    // Opened like every other volume file, so the journal this becomes has
    // the same mode and open flags as one that was never compacted.
    let file = open_volume_file(
        OpenOptions::new().write(true).create(true).truncate(true),
        tmp_path,
    )?;
    let mut w = BufWriter::new(file);
    let mut buf = [0u8; NEEDLE_ID_SIZE];
    for id in ids {
        id.to_bytes(&mut buf);
        w.write_all(&buf)?;
    }
    // into_inner flushes; into_error keeps the flush's own ErrorKind.
    w.into_inner()
        .map_err(io::IntoInnerError::into_error)?
        .sync_all()
}

impl EcVolume {
    /// Create a new EcVolume. Opens the .ecx index (required) and the .ecj journal.
    pub fn new(
        dir: &str,
        dir_idx: &str,
        collection: &str,
        volume_id: VolumeId,
    ) -> io::Result<Self> {
        Self::open_with(dir, dir_idx, collection, volume_id, EcjFsOps::REAL)
    }

    /// [`Self::new`] with the journal-compaction filesystem steps supplied, so
    /// tests can drive a failing publish through the real mount.
    fn open_with(
        dir: &str,
        dir_idx: &str,
        collection: &str,
        volume_id: VolumeId,
        ecj_ops: EcjFsOps,
    ) -> io::Result<Self> {
        // One load of the volume's `.vif`, used for both the shard config and
        // the version / dat-size fields below.
        let vif = load_vif_info(dir, dir_idx, collection, volume_id)?;
        // Both directories: a split -dir/-dir.idx layout keeps the .ecsum with
        // the INDEX, and it is the only layout record a volume whose vif is
        // absent or config-free still has.
        let (data_shards, parity_shards, block_size) =
            ec_shard_config_from(vif.as_ref(), &[dir, dir_idx], collection, volume_id)?;

        let total_shards = (data_shards + parity_shards) as usize;
        let mut shards = Vec::with_capacity(total_shards);
        for _ in 0..total_shards {
            shards.push(None);
        }

        // Read expire_at_sec, version, and dat_file_size from .vif if
        // present (matches Go's MaybeLoadVolumeInfo). Prefer the data
        // dir; fall back to the idx dir for the cross-disk reconcile
        // case (#9212 / #9244). `dat_file_size` is the source .dat
        // size at encode time, used both by `locate_ec_shard_needle`
        // for shard-size math and by the Store-level prune in
        // `store_ec_reconcile.rs` to verify a sibling-disk .dat is
        // plausibly the encoding source (#9478).
        let (expire_at_sec, vif_version, vif_dat_file_size, encode_ts_ns) =
            // Absent everywhere = legacy defaults; a present-but-unreadable or
            // malformed vif already failed the mount in load_vif_info above.
            match vif.as_ref() {
                Some(vif_info) => {
                    let ver = if vif_info.version > 0 {
                        Version(vif_info.version as u8)
                    } else {
                        Version::current()
                    };
                    let cfg_encode_ts_ns = vif_info
                        .ec_shard_config
                        .as_ref()
                        .map_or(0, |c| c.encode_ts_ns);
                    (
                        vif_info.expire_at_sec,
                        ver,
                        vif_info.dat_file_size,
                        cfg_encode_ts_ns,
                    )
                }
                None => (0, Version::current(), 0, 0),
            };

        let mut vol = EcVolume {
            volume_id,
            collection: collection.to_string(),
            dir: dir.to_string(),
            dir_idx: dir_idx.to_string(),
            version: vif_version,
            shards,
            dat_file_size: vif_dat_file_size,
            data_shards,
            parity_shards,
            block_size,
            ecx_file: None,
            ecx_file_size: 0,
            ecj_file: None,
            ecj_hold: None,
            ecj_file_size: 0,
            deleted_needles: RwLock::new(HashSet::new()),
            disk_type: DiskType::default(),
            ecx_actual_dir: dir_idx.to_string(),
            shard_locations: RwLock::new(ShardLocationCache::default()),
            expire_at_sec,
            encode_ts_ns,
            bitrot: None,
            bitrot_status: crate::storage::erasure_coding::ec_bitrot::BitrotStatus::Off,
            bitrot_source_dir: String::new(),
            io_errors: IoErrorTracker::default(),
        };

        // Open .ecx file (sorted index) in read/write mode for in-place deletion marking.
        // Matches Go which opens ecx for writing via MarkNeedleDeleted.
        //
        // Resolve it the way Go's NewEcVolume does: prefer a non-empty copy,
        // the one co-located with the shard data first, then the caller's
        // index directory — either the shared -dir.idx dir or a sibling disk
        // that owns the .ecx when this disk holds only a 0-byte stub left by an
        // interrupted copy. A 0-byte .ecx is also a legitimate empty index, so
        // it yields only to a non-empty copy elsewhere, never to a mere
        // absence. No .ecx at all fails the mount with NotFound (Go wraps
        // os.ErrNotExist): an EcVolume without an index would advertise shards
        // that no read can ever serve.
        let local_ecx = format!(
            "{}.ecx",
            crate::storage::volume::volume_file_name(dir, collection, volume_id)
        );
        let shared_ecx = vol.ecx_file_name();
        let local_size = ecx_file_size(&local_ecx);
        let shared_size = if dir_idx != dir {
            ecx_file_size(&shared_ecx)
        } else {
            None
        };
        let use_local = match (local_size, shared_size) {
            (Some(n), _) if n > 0 => true,
            (_, Some(n)) if n > 0 => {
                tracing::debug!(
                    volume_id = volume_id.0,
                    "ecx not local at {}, using {}",
                    local_ecx,
                    shared_ecx
                );
                false
            }
            // Only 0-byte copies exist: an empty index, local first.
            (Some(_), _) => true,
            (None, Some(_)) => false,
            (None, None) => {
                let tried = if dir_idx != dir {
                    format!("{} (or {})", local_ecx, shared_ecx)
                } else {
                    local_ecx
                };
                return Err(io::Error::new(
                    io::ErrorKind::NotFound,
                    format!("cannot open ec volume index {}: not found", tried),
                ));
            }
        };
        let ecx_path = if use_local {
            vol.ecx_actual_dir = dir.to_string();
            local_ecx
        } else {
            shared_ecx
        };
        let file = open_volume_file(OpenOptions::new().read(true).write(true), &ecx_path)?;
        vol.ecx_file_size = file.metadata()?.len() as i64;
        vol.ecx_file = Some(file);

        // Open .ecj file (deletion journal) — use ecx_actual_dir for consistency.
        // Note: Go does NOT replay .ecj into .ecx at volume load (RebuildEcxFile
        // is only invoked from specific decode/rebuild gRPC handlers), so we
        // don't either. Tombstones from prior sessions were already written
        // in-place in .ecx. The journal only grows at runtime; the one rewrite
        // at mount is `maybe_compact_ecj` below, which folds a bloated journal
        // down to the set it encodes when no other holder or writer can reach
        // the file.
        let ecj_base =
            crate::storage::volume::volume_file_name(&vol.ecx_actual_dir, collection, volume_id);
        let ecj_path = format!("{}.ecj", ecj_base);

        // Register as a holder before touching the file: this waits out a
        // compaction another disk's volume may be running on the same path, so
        // the tail repair and the handle below see the final inode, and it
        // stops any compaction from replacing the file under this handle.
        vol.ecj_hold = Some(EcjHold::acquire(&ecj_path));

        // Repair a torn tail BEFORE the append handle exists.
        //
        // The file is a flat array of fixed-size records and the journal handle
        // is in append mode, so every write lands at the physical end. A
        // trailing partial record therefore knocks every later append out of
        // alignment: the loader below skips the partial bytes, but the next
        // mount decodes those bytes together with the leading bytes of a real
        // entry, yielding one garbage id and silently dropping the delete that
        // followed the tear. Truncating to a whole number of records costs at
        // most one incomplete id that was never readable anyway.
        //
        // This deliberately uses its own read+write handle rather than the
        // append handle opened below: on Windows, `append(true)` requests
        // FILE_APPEND_DATA *without* FILE_WRITE_DATA (and `.write(true)` is
        // subsumed by `.append(true)`), so SetEndOfFile through that handle
        // fails with ERROR_ACCESS_DENIED. The handle is scoped so it is closed
        // again before the append handle opens.
        {
            let repair = open_volume_file(
                OpenOptions::new().read(true).write(true).create(true),
                &ecj_path,
            )?;
            let on_disk = repair.metadata()?.len() as i64;
            let ragged = on_disk % NEEDLE_ID_SIZE as i64;
            if ragged != 0 {
                let whole = on_disk - ragged;
                tracing::warn!(
                    volume_id = volume_id.0,
                    collection = %collection,
                    on_disk_bytes = on_disk,
                    truncated_to = whole,
                    "truncating torn .ecj tail so later appends stay aligned",
                );
                repair.set_len(whole as u64)?;
                repair.sync_all()?;
            }
        }

        let ecj_file = open_ecj_append(&ecj_path)?;
        vol.ecj_file_size = ecj_file.metadata()?.len() as i64;
        vol.ecj_file = Some(ecj_file);

        // Seed the in-memory deleted set from the journal.
        vol.load_deleted_needles_from_ecj(ecj_ops.read_at)?;

        // Load the generation-0 EC bitrot checksum sidecar. Optional, except
        // when it contradicts the volume's own geometry — see
        // load_bitrot_for_generation.
        vol.load_active_bitrot_sidecar(&[])?;

        // Last, once every check that can refuse the mount has passed: a
        // volume the server declines to serve keeps its files as they were.
        vol.maybe_compact_ecj(ecj_ops).map_err(|e| {
            io::Error::new(
                e.kind(),
                format!(
                    "ec volume {}: .ecj compaction left no usable journal handle: {}",
                    volume_id.0, e
                ),
            )
        })?;

        Ok(vol)
    }

    /// Re-resolve the checksum sidecar for a volume that is already mounted. A
    /// shard delivery can bring the manifest with it, and the receive path only
    /// writes the file, so without this the in-memory volume keeps the
    /// protection state it resolved at mount (off) until a remount.
    pub fn reload_bitrot_sidecar(&mut self, additional_dirs: &[String]) {
        if let Err(e) = self.load_active_bitrot_sidecar(additional_dirs) {
            tracing::warn!(
                volume_id = self.volume_id.0,
                error = %e,
                "reload bitrot sidecar",
            );
        }
    }

    /// Load the generation-0 checksum sidecar into `self.bitrot`/`self.bitrot_status`.
    /// OSS only produces generation-0 (fresh-encode) sidecars, mirroring Go's
    /// `loadActiveBitrotSidecar`.
    fn load_active_bitrot_sidecar(&mut self, additional_dirs: &[String]) -> io::Result<()> {
        self.load_bitrot_for_generation(0, additional_dirs)
    }

    /// Load and validate the sidecar describing `generation`, setting
    /// `self.bitrot`/`self.bitrot_status`. Absent or generation/config-mismatched
    /// => `Off` (protection off, not corruption); self-integrity or manifest
    /// failure => `Invalid` with a warning (protection off pending repair); usable
    /// => `On`. Mirrors Go's `loadBitrotForGeneration`.
    fn load_bitrot_for_generation(
        &mut self,
        generation: u32,
        additional_dirs: &[String],
    ) -> io::Result<()> {
        use crate::storage::erasure_coding::ec_bitrot;
        // Data base then index base, matching Go's findBitrotSidecar. A split
        // -dir/-dir.idx location keeps the sidecar with the INDEX, and on a
        // multi-disk server sharing one index directory that is the only copy
        // the per-disk runtimes other than the first can see — searching the
        // data base alone left them reporting no protection at all.
        let data_path = ec_bitrot::bitrot_sidecar_path(&self.base_name(), generation);
        let idx_path = ec_bitrot::bitrot_sidecar_path(&self.idx_base_name(), generation);
        // Sibling disks last: startup mirroring gives each shard-bearing disk
        // its own .ecx/.ecj/.vif but not the sidecar, and a delivery lands
        // exactly one copy, so a runtime restricted to its own two directories
        // would report no protection however often it reloaded.
        let path = std::iter::once(data_path.clone())
            .chain(std::iter::once(idx_path))
            .chain(additional_dirs.iter().filter(|d| !d.is_empty()).map(|d| {
                ec_bitrot::bitrot_sidecar_path(
                    &crate::storage::volume::volume_file_name(d, &self.collection, self.volume_id),
                    generation,
                )
            }))
            .find(|p| std::path::Path::new(p).exists())
            .unwrap_or(data_path);
        let loaded = ec_bitrot::load_bitrot_sidecar(&path);
        // A sidecar written for THIS generation that contradicts the volume's
        // geometry is not "no protection" — it says the layout the volume is
        // about to serve reads with is wrong. Fail the mount.
        if let Ok(prot) = &loaded
            && prot.generation == generation
            && !ec_bitrot::geometry_matches(
                prot,
                self.data_shards as usize,
                self.parity_shards as usize,
                self.block_size,
            )
        {
            let cfg = prot.ec_shard_config.as_ref();
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!(
                    "ec volume {} generation {}: {} records layout {}+{} block {} but the volume is mounted as {}+{} block {}; refusing to serve one of the two layouts",
                    self.volume_id.0,
                    generation,
                    path,
                    cfg.map(|c| c.data_shards).unwrap_or(0),
                    cfg.map(|c| c.parity_shards).unwrap_or(0),
                    cfg.map(|c| c.block_size).unwrap_or(0),
                    self.data_shards,
                    self.parity_shards,
                    self.block_size,
                ),
            ));
        }
        let status = ec_bitrot::resolve_status(
            &loaded,
            generation,
            self.data_shards as usize,
            self.parity_shards as usize,
        );
        // Provenance is committed together with the sidecar it describes, so a
        // swallowed reload (`reload_bitrot_sidecar` logs and discards the `Err`
        // above) cannot leave them disagreeing: the geometry-mismatch return
        // happens before this point, so `bitrot`/`bitrot_status` below and
        // `bitrot_source_dir` here always describe the same load attempt.
        // Only a path that actually exists is a real source: `path` falls back
        // to the data path when nothing was found, and recording that would
        // claim a provenance the volume does not have.
        self.bitrot_source_dir = if std::path::Path::new(&path).exists() {
            std::path::Path::new(&path)
                .parent()
                .map(|p| p.to_string_lossy().into_owned())
                .unwrap_or_default()
        } else {
            String::new()
        };
        self.bitrot = None;
        self.bitrot_status = status;
        match status {
            ec_bitrot::BitrotStatus::On => self.bitrot = loaded.ok(),
            ec_bitrot::BitrotStatus::Off => {}
            ec_bitrot::BitrotStatus::Invalid => {
                tracing::warn!(
                    volume_id = self.volume_id.0,
                    path = %path,
                    generation,
                    "ec volume: bitrot sidecar present but invalid; protection off pending repair",
                );
            }
        }
        Ok(())
    }

    /// The active-generation bitrot protection AND its status (cached at mount),
    /// mirroring Go's `EcVolume.BitrotProtection()`. Preserves the distinction
    /// `checksum_scrub` needs: an absent/generation-mismatched sidecar is `Off`
    /// (a clean no-op), a present-but-malformed one is `Invalid` (a real integrity
    /// error). Returns `Some(prot)` only for `On`.
    pub(crate) fn bitrot_protection(
        &self,
    ) -> (
        Option<crate::pb::volume_server_pb::EcBitrotProtection>,
        crate::storage::erasure_coding::ec_bitrot::BitrotStatus,
    ) {
        (self.bitrot.clone(), self.bitrot_status)
    }

    /// Read-only EC bitrot checksum scrub of the LOCAL shards of this volume's
    /// active generation.
    ///
    /// Loads the active-generation `.ecsum` sidecar and, for each locally-held
    /// shard, reads the on-disk shard file in `block_size` chunks and compares
    /// each block's CRC32C against the sidecar. Returns
    /// `(blocks_scanned, mismatched_shards, errors)`.
    ///
    /// If more than `parity_shards` shards mismatch wholesale (i.e. every block
    /// of those shards is wrong — the signature of a stale/wrong sidecar rather
    /// than localized disk rot), the result is classified as a suspect sidecar:
    /// `mismatched_shards` is cleared and an integrity note is added to `errors`
    /// instead. A genuine multi-shard disk failure of that magnitude is
    /// already unrecoverable, so treating it as a sidecar-integrity issue avoids
    /// raising false shard-corruption alarms.
    ///
    /// A third outcome sits ahead of both of the above: if the identity fence
    /// excluded a runtime AND the sidecar that supplied the protection was
    /// resolved from that excluded runtime's directory, the checksums cannot be
    /// trusted against
    /// the merged shards at all. That case returns `(0, [], [errors])` with an
    /// "unverifiable" note and never touches a shard handle — no scan, no
    /// wholesale-mismatch classification, no blamed shard.
    ///
    /// This method NEVER deletes or mutates anything — it is purely diagnostic.
    /// Duplicates the mounted shard handles so `run()` can scan with the store
    /// guard released; see `EcChecksumScrubPlan` for why that matters.
    pub fn checksum_scrub_plan(&self) -> EcChecksumScrubPlan {
        EcChecksumScrubPlan::for_volumes(&[self]).expect("a single runtime is never an empty slice")
    }

    /// Convenience wrapper preserving the original call shape. Callers that
    /// hold the store lock MUST use `checksum_scrub_plan()` + `run()` instead.
    pub fn checksum_scrub(&self) -> (u64, Vec<u32>, Vec<String>) {
        self.checksum_scrub_plan().run()
    }

    /// Walk the .ecj journal and populate `deleted_needles`. Called once
    /// from `new()` under exclusive ownership of the just-constructed
    /// EcVolume, so locking is not strictly required — but we take the
    /// write lock anyway for symmetry with later mutations.
    ///
    /// Read in large chunks. This previously issued one `NEEDLE_ID_SIZE`-byte
    /// positional read per entry, which is fine for a healthy journal (a few
    /// KB) and catastrophic for a bloated one: at the 1.51 TB seen in
    /// production that is ~188e9 syscalls, so the server spun at 100% of one
    /// core with a 31 MB RSS — the set stays small because the ids repeat —
    /// and never opened its HTTP port, which made the master unregister every
    /// volume it held. Chunked reads cut that by ~`ECJ_LOAD_CHUNK_BYTES / 8`.
    fn load_deleted_needles_from_ecj(
        &mut self,
        read_at: fn(&File, &mut [u8], u64) -> io::Result<()>,
    ) -> io::Result<()> {
        let ecj_file = match self.ecj_file.as_ref() {
            Some(f) => f,
            None => return Ok(()),
        };
        if self.ecj_file_size < NEEDLE_ID_SIZE as i64 {
            return Ok(());
        }

        // Build into a local set and merge at the end. The previous version
        // held the `deleted_needles` write lock for the whole scan, which on a
        // bloated journal is the entire (unbounded) startup.
        let mut loaded: HashSet<NeedleId> = HashSet::new();
        read_ecj_ids_with(read_at, ecj_file, 0, self.ecj_file_size as u64, &mut loaded)?;

        let mut set = self
            .deleted_needles
            .write()
            .map_err(|_| io::Error::other("deleted_needles lock poisoned"))?;
        set.extend(loaded);
        Ok(())
    }

    /// Rewrite a bloated `.ecj` from the set just loaded out of it.
    ///
    /// The journal is semantically a SET of deleted needle ids, but it is
    /// written as an append-only log that nothing dedupes or truncates. Several
    /// paths append a peer's *entire* journal onto this one —
    /// `VolumeEcShardsCopy` (`copy_ecj_file`), EC index recovery, and
    /// `ec_decode`'s deliberate merge across holders — so a volume whose shards
    /// are repeatedly balanced between two servers grows the file geometrically.
    /// Observed in production: 1.51 TB and 1.30 TB for one volume holding ~100
    /// distinct ids, which wedged both owning volume servers at startup.
    ///
    /// Compaction is safe because the set IS the journal's meaning, provided
    /// nothing else can write the file between the load and the rename. That is
    /// what the `ecj_registry` reservation and the on-disk re-check establish:
    /// this volume is the only holder of the path, no copy is writing to it, and
    /// the file is still the inode and size that was loaded. Crash-safe via temp
    /// file + fsync + rename + directory fsync.
    ///
    /// Returns `Err` only when the volume is left without a usable journal
    /// handle, which must fail the mount. Every other failure leaves the
    /// original journal in place and is logged here.
    fn maybe_compact_ecj(&mut self, ops: EcjFsOps) -> io::Result<()> {
        let ecj_path = self.ecj_file_name();
        let tmp_path = format!("{}{}", self.idx_base_name(), ECJ_COMPACT_TMP_EXT);
        let wanted = self.ecj_needs_compaction()?;
        let stale_tmp = std::path::Path::new(&tmp_path).exists();
        if !wanted && !stale_tmp {
            return Ok(());
        }
        let Some(_reservation) = self
            .ecj_hold
            .as_ref()
            .and_then(EcjHold::try_begin_compaction)
        else {
            tracing::debug!(
                volume_id = self.volume_id.0,
                path = %ecj_path,
                "skipping .ecj compaction: another holder or a copy can reach the journal",
            );
            return Ok(());
        };
        // A tmp left by a crash between its write and the rename. Removed under
        // the reservation, so it cannot be another holder's compaction in flight.
        if stale_tmp {
            let _ = fs::remove_file(&tmp_path);
        }
        if !wanted {
            return Ok(());
        }
        self.compact_ecj_reserved(&ecj_path, &tmp_path, ops)
    }

    /// The part of [`Self::maybe_compact_ecj`] that runs under the path
    /// reservation: re-check the file, write the compacted tmp, publish it.
    /// Same error contract: `Err` only when no usable journal handle is left.
    fn compact_ecj_reserved(
        &mut self,
        ecj_path: &str,
        tmp_path: &str,
        ops: EcjFsOps,
    ) -> io::Result<()> {
        match self.ecj_unchanged_since_load(ecj_path) {
            Ok(true) => {}
            Ok(false) => {
                tracing::warn!(
                    volume_id = self.volume_id.0,
                    path = %ecj_path,
                    "skipping .ecj compaction: the journal changed on disk after it was loaded",
                );
                return Ok(());
            }
            Err(e) => {
                tracing::warn!(volume_id = self.volume_id.0, error = %e, "compact .ecj: stat journal");
                return Ok(());
            }
        }
        let ids = self.sorted_deleted_ids()?;
        tracing::warn!(
            volume_id = self.volume_id.0,
            collection = %self.collection,
            on_disk_bytes = self.ecj_file_size,
            unique_ids = ids.len(),
            compacted_bytes = ids.len() * NEEDLE_ID_SIZE,
            "compacting bloated .ecj deletion journal",
        );
        if let Err(e) = write_compacted_ecj_tmp(tmp_path, &ids) {
            // Leaving the partial temp file behind would pin its bytes for the
            // life of the process — and the likeliest reason to land here is
            // ENOSPC, where those bytes are exactly what is scarce.
            let _ = fs::remove_file(tmp_path);
            tracing::warn!(volume_id = self.volume_id.0, error = %e, "compact .ecj: write compacted journal");
            return Ok(());
        }
        match self.publish_compacted_ecj(ecj_path, tmp_path, ops) {
            Ok(()) => Ok(()),
            Err(EcjPublishError::Abandoned(e)) => {
                tracing::warn!(volume_id = self.volume_id.0, error = %e, "compact .ecj: journal left as it was");
                Ok(())
            }
            Err(EcjPublishError::HandleLost(e)) => Err(e),
        }
    }

    /// Whether the loaded journal is bloated enough to rewrite:
    /// `file_bytes >= max(ECJ_COMPACT_MIN_BYTES, ECJ_COMPACT_RATIO * set_bytes)`.
    /// The 1 MiB floor keeps a small healthy journal from ever being rewritten.
    /// Reads only the set's length, so the common no-op mount copies nothing.
    fn ecj_needs_compaction(&self) -> io::Result<bool> {
        let distinct = self
            .deleted_needles
            .read()
            .map_err(|_| io::Error::other("deleted_needles lock poisoned"))?
            .len();
        let compacted_len = (distinct as i64).saturating_mul(NEEDLE_ID_SIZE as i64);
        Ok(self.ecj_file_size >= ECJ_COMPACT_MIN_BYTES
            && self.ecj_file_size >= compacted_len.saturating_mul(ECJ_COMPACT_RATIO))
    }

    /// The deleted set, sorted so the rewritten file is deterministic: two
    /// holders that compact the same set produce byte-identical journals, which
    /// makes a difference between them meaningful rather than ordering noise.
    fn sorted_deleted_ids(&self) -> io::Result<Vec<NeedleId>> {
        let mut ids: Vec<NeedleId> = self
            .deleted_needles
            .read()
            .map_err(|_| io::Error::other("deleted_needles lock poisoned"))?
            .iter()
            .copied()
            .collect();
        ids.sort_unstable();
        Ok(ids)
    }

    /// Whether the file at `ecj_path` is still the one this volume loaded: the
    /// same inode as the open handle (Unix) and the size the set was read from.
    /// A copy that appended, or replaced the file, after the load fails this,
    /// and compacting then would drop what it wrote.
    fn ecj_unchanged_since_load(&self, ecj_path: &str) -> io::Result<bool> {
        let Some(held) = self.ecj_file.as_ref() else {
            return Ok(false);
        };
        let held = held.metadata()?;
        let on_path = fs::metadata(ecj_path)?;
        #[cfg(unix)]
        {
            use std::os::unix::fs::MetadataExt;
            if held.dev() != on_path.dev() || held.ino() != on_path.ino() {
                return Ok(false);
            }
        }
        Ok(held.len() == on_path.len() && on_path.len() as i64 == self.ecj_file_size)
    }

    /// Publish a compacted tmp file over the live journal: drop the handle
    /// (Windows cannot rename over an open file), rename, fsync the directory,
    /// reopen the append handle.
    ///
    /// A failed rename publishes nothing: the tmp is removed and the handle to
    /// the original journal restored, giving [`EcjPublishError::Abandoned`]. If
    /// that restore fails too, or anything fails after the rename, the volume
    /// has no usable handle: [`EcjPublishError::HandleLost`], carrying every
    /// error involved.
    fn publish_compacted_ecj(
        &mut self,
        ecj_path: &str,
        tmp_path: &str,
        ops: EcjFsOps,
    ) -> Result<(), EcjPublishError> {
        // Drop our handle on the destination BEFORE replacing it. On Windows a
        // rename over a file that is still open fails outright, which would
        // make compaction a permanent no-op there; on Unix the handle would
        // survive as an unlinked inode, which is worse than useless.
        self.ecj_file = None;
        if let Err(rename_err) = (ops.rename)(tmp_path, ecj_path) {
            let _ = fs::remove_file(tmp_path);
            return match (ops.reopen)(ecj_path) {
                Ok(f) => {
                    self.ecj_file = Some(f);
                    Err(EcjPublishError::Abandoned(rename_err))
                }
                Err(reopen_err) => Err(EcjPublishError::HandleLost(io::Error::new(
                    reopen_err.kind(),
                    format!(
                        "rename {} over {}: {}; reopening the original journal then failed: {}",
                        tmp_path, ecj_path, rename_err, reopen_err
                    ),
                ))),
            };
        }
        // Syncing the temp file persisted its contents, not the directory entry
        // that now points at it. Without this a power loss can restore the old
        // bloated journal — and, worse, discard deletes that were acknowledged
        // against the replacement in between. Matches the other replace-by-rename
        // paths in this crate (ec_decoder, volume_idx_rebuild, volume_idx_repair).
        let lost = |what: &str, e: io::Error| {
            EcjPublishError::HandleLost(io::Error::new(
                e.kind(),
                format!("replaced {} but could not {}: {}", ecj_path, what, e),
            ))
        };
        (ops.fsync_dir)(ecj_path).map_err(|e| lost("fsync its directory", e))?;
        // Reopen so later journal_delete appends land in the file just written.
        let reopened = (ops.reopen)(ecj_path).map_err(|e| lost("reopen it", e))?;
        let size = reopened.metadata().map_err(|e| lost("stat it", e))?.len();
        self.ecj_file_size = size as i64;
        self.ecj_file = Some(reopened);
        Ok(())
    }

    /// Returns (file_count, delete_count) for this EC volume. Mirrors Go's
    /// `EcVolume.FileAndDeleteCount`:
    ///
    ///   file_count   = ecx_file_size / NEEDLE_MAP_ENTRY_SIZE  — total
    ///                  entries in the sealed sorted .ecx index.
    ///   delete_count = deleted_needles.len()                  — unique
    ///                  runtime deletes tracked in memory (seeded from
    ///                  .ecj on load and updated by `journal_delete`).
    ///
    /// Because each needle delete is applied on exactly one shard holder,
    /// the admin aggregation sums delete_count across nodes while taking
    /// file_count from a single holder (they are identical per volume).
    pub fn file_and_delete_count(&self) -> (u64, u64) {
        let file_count = (self.ecx_file_size as u64) / (NEEDLE_MAP_ENTRY_SIZE as u64);
        let delete_count = self
            .deleted_needles
            .read()
            .map(|s| s.len() as u64)
            .unwrap_or(0);
        (file_count, delete_count)
    }

    /// Reports whether the given needle id is in the in-memory deleted set.
    pub fn is_needle_deleted(&self, needle_id: NeedleId) -> bool {
        self.deleted_needles
            .read()
            .map(|s| s.contains(&needle_id))
            .unwrap_or(false)
    }

    // ---- File names ----

    fn base_name(&self) -> String {
        crate::storage::volume::volume_file_name(&self.dir, &self.collection, self.volume_id)
    }

    /// Base path for the .ecx / .ecj index pair. Resolved from
    /// `ecx_actual_dir` (initialized to `dir_idx` and only updated after a
    /// successful idx-dir → data-dir fallback in `new()`), so every call site
    /// agrees on the same file regardless of whether the fallback fired.
    fn idx_base_name(&self) -> String {
        crate::storage::volume::volume_file_name(
            &self.ecx_actual_dir,
            &self.collection,
            self.volume_id,
        )
    }

    pub fn ecx_file_name(&self) -> String {
        format!("{}.ecx", self.idx_base_name())
    }

    pub fn ecj_file_name(&self) -> String {
        format!("{}.ecj", self.idx_base_name())
    }

    /// Sync the EC volume's journal and index files to disk (matching Go's ecv.Sync()).
    /// Go flushes both .ecj and .ecx to ensure in-place deletion marks are persisted.
    pub fn sync_to_disk(&self) -> io::Result<()> {
        if let Some(ref ecj_file) = self.ecj_file {
            ecj_file.sync_all()?;
        }
        if let Some(ref ecx_file) = self.ecx_file {
            ecx_file.sync_all()?;
        }
        Ok(())
    }

    // ---- Shard management ----

    /// Add a shard to this volume.
    pub fn add_shard(&mut self, mut shard: EcVolumeShard) -> io::Result<()> {
        let id = shard.shard_id as usize;
        let total_shards = (self.data_shards + self.parity_shards) as usize;
        if id >= total_shards {
            return Err(io::Error::new(
                io::ErrorKind::InvalidInput,
                format!("invalid shard id: {} (max {})", id, total_shards - 1),
            ));
        }
        shard.open()?;
        // A 0-byte shard file beside an index with entries is residue of a
        // failed copy or a truncation, not a mountable shard: registering
        // it would advertise a size-0 claim that serves nothing and, since
        // placement pins re-copies to the owning disk, would keep
        // attracting repairs to a file that was never valid. A 0-byte shard
        // beside a 0-byte index is different — that is the legitimate
        // layout of a volume encoded with no live needles, and it must keep
        // mounting. The startup scan already skips 0-byte shard files; this
        // covers the mount path. Mirrors AddEcVolumeShard in
        // weed/storage/erasure_coding/ec_volume.go.
        if shard.file_size() == 0 && self.ecx_file_size > 0 {
            let path = shard.file_name();
            shard.close();
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!(
                    "ec volume shard {} is empty (0 bytes) but the index has entries: residue of a failed copy, not a mountable shard",
                    path
                ),
            ));
        }
        self.shards[id] = Some(shard);
        Ok(())
    }

    /// Override the disk type the EC volume (and its already-mounted
    /// shards) reports under. Used by the `VolumeEcShardsMount` handler
    /// so the source volume's disk type is preserved across encoding
    /// (#9423). Not persisted across restarts — disk-scan reload paths
    /// default to the physical location's disk type.
    pub fn set_disk_type(&mut self, d: DiskType) {
        self.disk_type = d.clone();
        for shard in self.shards.iter_mut().flatten() {
            shard.disk_type = d.clone();
        }
    }

    /// Remove and close a shard.
    pub fn remove_shard(&mut self, shard_id: ShardId) -> io::Result<()> {
        let idx = shard_id as usize;
        if idx >= self.shards.len() {
            return Err(io::Error::new(
                io::ErrorKind::InvalidInput,
                format!(
                    "invalid shard id {} (max {})",
                    shard_id,
                    self.shards.len().saturating_sub(1)
                ),
            ));
        }
        if let Some(ref mut shard) = self.shards[idx] {
            shard.close();
        }
        self.shards[idx] = None;
        Ok(())
    }

    /// Get a ShardBits bitmap of locally available shards.
    pub fn shard_bits(&self) -> ShardBits {
        let mut bits = ShardBits::default();
        for (i, shard) in self.shards.iter().enumerate() {
            if shard.is_some() {
                bits.add_shard_id(i as ShardId);
            }
        }
        bits
    }

    /// Count of locally available shards.
    pub fn shard_count(&self) -> usize {
        self.shards.iter().filter(|s| s.is_some()).count()
    }

    /// Reports whether `shard_id` is currently registered to this
    /// EcVolume (used by the cross-disk reconcile to skip already-
    /// loaded shards).
    pub fn has_shard(&self, shard_id: ShardId) -> bool {
        self.shards
            .get(shard_id as usize)
            .map(|s| s.is_some())
            .unwrap_or(false)
    }

    /// Directory where this EcVolume's `.ecx` was actually opened
    /// (may differ from `dir_idx` when the legacy "written before
    /// -dir.idx was set" fallback or the cross-disk reconcile path
    /// pointed it elsewhere).
    pub fn ecx_actual_dir(&self) -> &str {
        &self.ecx_actual_dir
    }

    pub fn is_time_to_destroy(&self) -> bool {
        self.expire_at_sec > 0
            && SystemTime::now()
                .duration_since(UNIX_EPOCH)
                .unwrap_or_default()
                .as_secs()
                > self.expire_at_sec
    }

    pub fn to_volume_ec_shard_information_messages(
        &self,
        disk_id: u32,
    ) -> Vec<master_pb::VolumeEcShardInformationMessage> {
        let mut ec_index_bits: u32 = 0;
        let mut shard_sizes = Vec::new();
        for shard in self.shards.iter().flatten() {
            ec_index_bits |= 1u32 << shard.shard_id;
            shard_sizes.push(shard.file_size());
        }

        if ec_index_bits == 0 {
            return Vec::new();
        }

        let (file_count, delete_count) = self.file_and_delete_count();

        vec![master_pb::VolumeEcShardInformationMessage {
            id: self.volume_id.0,
            collection: self.collection.clone(),
            ec_index_bits,
            shard_sizes,
            disk_type: self.disk_type.to_string(),
            expire_at_sec: self.expire_at_sec,
            disk_id,
            file_count,
            delete_count,
            encode_ts_ns: self.encode_ts_ns,
            ..Default::default()
        }]
    }

    // ---- Shard locations (distributed tracking) ----

    /// Merge a fresh `LookupEcVolume` reply into the shard-locations cache and
    /// stamp the refresh time, returning a clone of the resulting map.
    ///
    /// Per-shard upsert (mirrors Go's `cachedLookupEcShardLocations`): each shard
    /// id present in the reply overwrites its cached entry, while shard ids absent
    /// from the reply keep their previously-cached locations — so a reply that
    /// passes the data-shard completeness guard but omits a shard already in cache
    /// does not drop that shard's known location (unlike a full replace).
    ///
    /// Map and refresh time advance in one write section, so no reader can pair
    /// the merged map with the previous lookup's timestamp.
    pub(crate) fn merge_shard_locations(
        &self,
        locations: HashMap<ShardId, Vec<String>>,
    ) -> HashMap<ShardId, Vec<String>> {
        let mut cache = self.shard_locations.write().unwrap();
        for (shard_id, addrs) in locations {
            cache.locations.insert(shard_id, addrs);
        }
        cache.refreshed_at = Some(Instant::now());
        cache.locations.clone()
    }

    /// The cached map and the time it was learned, read in one section. Both
    /// halves describe the same refresh, which is what the freshness heuristic
    /// assumes when it ages the map against the timestamp.
    pub(crate) fn shard_locations_snapshot(
        &self,
    ) -> (HashMap<ShardId, Vec<String>>, Option<Instant>) {
        let cache = self.shard_locations.read().unwrap();
        (cache.locations.clone(), cache.refreshed_at)
    }

    /// Mark the cached map for a prompt re-check after a read failed against one
    /// of its locations.
    pub(crate) fn mark_shard_locations_stale(&self) {
        self.shard_locations.write().unwrap().stale = true;
    }

    /// Put the stale mark to `decide`, and clear it only if `decide` says the
    /// master lookup is going ahead — both in one critical section, so the mark
    /// is consumed exactly once by the refresh that answers for it and a mark
    /// raised meanwhile survives for the next one. `decide` owns the rest of the
    /// freshness rule; it judges the map its caller will actually read from,
    /// which is not necessarily the one cached here by the time it runs.
    ///
    /// `decide` runs under the cache write lock and must not acquire other locks:
    /// every caller reaches this while already holding the store read lock, so a
    /// cache-write → store-read inside `decide` would invert that order.
    pub(crate) fn claim_shard_locations_refresh(&self, decide: impl FnOnce(bool) -> bool) -> bool {
        let mut cache = self.shard_locations.write().unwrap();
        let refresh = decide(cache.stale);
        if refresh {
            cache.stale = false;
        }
        refresh
    }

    // ---- I/O error tracking ----

    pub fn check_read_write_error(&self, err: Option<&io::Error>) {
        self.io_errors.check_read_write_error(err);
    }

    pub fn get_io_error_state(&self) -> (Option<String>, i32, bool) {
        self.io_errors.get_io_error_state()
    }

    pub fn should_quarantine(&self) -> bool {
        self.io_errors.should_quarantine()
    }

    pub fn mark_io_quarantined(&self) {
        self.io_errors.mark_io_quarantined();
    }

    pub fn reset_io_error_state(&self) {
        self.io_errors.reset_io_error_state();
    }

    // ---- Index operations ----

    /// Find a needle's offset and size in the sorted .ecx index via binary search.
    pub fn find_needle_from_ecx(&self, needle_id: NeedleId) -> io::Result<Option<(Offset, Size)>> {
        let ecx_file = self
            .ecx_file
            .as_ref()
            .ok_or_else(|| io::Error::other("ecx file not open"))?;

        let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
        if entry_count == 0 {
            return Ok(None);
        }

        // Binary search
        let mut lo: usize = 0;
        let mut hi: usize = entry_count;
        let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];

        while lo < hi {
            let mid = lo + (hi - lo) / 2;
            let file_offset = (mid * NEEDLE_MAP_ENTRY_SIZE) as u64;

            // Positional read so concurrent find_needle_from_ecx calls on the
            // shared .ecx handle don't interleave a seek and a read and corrupt
            // each other's binary search.
            if let Err(e) = read_exact_at(ecx_file, &mut entry_buf, file_offset) {
                self.check_read_write_error(Some(&e));
                return Err(e);
            }

            let (key, offset, size) = idx_entry_from_bytes(&entry_buf);
            if key == needle_id {
                // Apply runtime deletion state on top of the sealed .ecx
                // lookup: a needle in the in-memory deleted set is
                // reported with TOMBSTONE_FILE_SIZE even though the .ecx
                // record itself is untouched.
                if self.is_needle_deleted(needle_id) {
                    self.check_read_write_error(None);
                    return Ok(Some((offset, TOMBSTONE_FILE_SIZE)));
                }
                self.check_read_write_error(None);
                return Ok(Some((offset, size)));
            } else if key < needle_id {
                lo = mid + 1;
            } else {
                hi = mid;
            }
        }

        self.check_read_write_error(None);
        Ok(None)
    }

    /// Large-block length of this volume's shard layout (the uniform block
    /// size when set, the legacy 1GiB otherwise). Mirrors Go's
    /// ECContext.LargeBlockSize.
    pub fn large_block_size(&self) -> i64 {
        if self.block_size > 0 {
            self.block_size
        } else {
            ERASURE_CODING_LARGE_BLOCK_SIZE as i64
        }
    }

    /// Small-block length of this volume's shard layout.
    pub fn small_block_size(&self) -> i64 {
        if self.block_size > 0 {
            self.block_size
        } else {
            ERASURE_CODING_SMALL_BLOCK_SIZE as i64
        }
    }

    /// Locate the EC shard intervals needed to read a needle.
    /// Locate the EC shard intervals covering a needle at `actual_offset` whose
    /// index size is `size`. Mirrors Go's EcVolume.LocateEcShardNeedleInterval.
    pub fn locate_ec_shard_needle_interval(
        &self,
        actual_offset: i64,
        size: Size,
    ) -> Vec<ec_locate::Interval> {
        // shardSize = datFileSize / DataShards when known, else ecdFileSize - 1
        // (shards are padded to the small block size; the -1 avoids an
        // off-by-one in the large-block row count).
        let shard_size = if self.dat_file_size > 0 {
            self.dat_file_size / self.data_shards as i64
        } else {
            self.shard_file_size() - 1
        };
        // locate_data wants the on-disk size (header+body+checksum+timestamp+padding).
        let actual = get_actual_size(size, self.version);
        ec_locate::locate_data(
            actual_offset,
            Size(actual as i32),
            shard_size,
            self.data_shards,
            self.large_block_size(),
            self.small_block_size(),
        )
    }

    /// Resolve an interval against this volume's shard block layout. Mirrors
    /// Go's EcVolume.IntervalToShardIdAndOffset.
    pub fn interval_to_shard_id_and_offset(
        &self,
        interval: &ec_locate::Interval,
    ) -> (ShardId, i64) {
        interval.to_shard_id_and_offset(
            self.data_shards,
            self.large_block_size(),
            self.small_block_size(),
        )
    }

    pub fn locate_needle(
        &self,
        needle_id: NeedleId,
    ) -> io::Result<Option<(Offset, Size, Vec<ec_locate::Interval>)>> {
        let (offset, size) = match self.find_needle_from_ecx(needle_id)? {
            Some((o, s)) => (o, s),
            None => return Ok(None),
        };

        if size.is_deleted() || offset.is_zero() {
            return Ok(None);
        }

        let intervals = self.locate_ec_shard_needle_interval(offset.to_actual_offset(), size);
        Ok(Some((offset, size, intervals)))
    }

    /// Read a full needle from locally available EC shards.
    ///
    /// Locates the needle in the .ecx index, determines which shard intervals
    /// contain its data, reads from local shards, and parses the result into
    /// a fully populated Needle (including last_modified, checksum, ttl).
    ///
    /// Returns `Ok(None)` if the needle is not found or is deleted.
    /// Returns an error if a required shard is not available locally.
    pub fn read_ec_shard_needle(&self, needle_id: NeedleId) -> io::Result<Option<Needle>> {
        let (offset, size, intervals) = match self.locate_needle(needle_id)? {
            Some(v) => v,
            None => return Ok(None),
        };

        if intervals.is_empty() {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                "no intervals for needle",
            ));
        }

        // Compute the total bytes we need to read (full needle on disk)
        let actual_size = get_actual_size(size, self.version) as usize;
        let mut bytes = Vec::with_capacity(actual_size);

        for interval in &intervals {
            let (shard_id, shard_offset) = self.interval_to_shard_id_and_offset(interval);
            let shard = self
                .shards
                .get(shard_id as usize)
                .and_then(|s| s.as_ref())
                .ok_or_else(|| {
                    io::Error::new(
                        io::ErrorKind::NotFound,
                        format!("ec shard {} not available locally", shard_id),
                    )
                })?;

            let mut buf = vec![0u8; interval.size as usize];
            let n = shard.read_at(&mut buf, shard_offset as u64)?;
            if n != buf.len() {
                return Err(io::Error::new(
                    io::ErrorKind::UnexpectedEof,
                    format!(
                        "short read on ec shard {}: read {} of {} bytes for needle {}",
                        shard_id,
                        n,
                        buf.len(),
                        needle_id
                    ),
                ));
            }
            bytes.extend_from_slice(&buf);
        }

        // Truncate to exact actual_size (intervals may span more than needed)
        bytes.truncate(actual_size);

        if bytes.len() < actual_size {
            return Err(io::Error::new(
                io::ErrorKind::UnexpectedEof,
                format!(
                    "read {} bytes but need {} for needle {}",
                    bytes.len(),
                    actual_size,
                    needle_id
                ),
            ));
        }

        let mut n = Needle::default();
        n.read_bytes(&bytes, offset.to_actual_offset(), size, self.version)
            .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("{}", e)))?;

        Ok(Some(n))
    }

    /// Get the size of a single shard (all shards are the same size).
    fn shard_file_size(&self) -> i64 {
        if let Some(s) = self.shards.iter().flatten().next() {
            return s.file_size();
        }
        0
    }

    /// Walk the .ecx index and return (file_count, file_deleted_count, total_size).
    /// total_size sums size.Raw() for all entries (including deleted), matching Go's WalkIndex.
    pub fn walk_ecx_stats(&self) -> io::Result<(u64, u64, u64)> {
        let ecx_file = match self.ecx_file.as_ref() {
            Some(f) => f,
            None => return Ok((0, 0, 0)),
        };

        let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
        let mut files: u64 = 0;
        let mut files_deleted: u64 = 0;
        let mut total_size: u64 = 0;
        let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];

        for i in 0..entry_count {
            let file_offset = (i * NEEDLE_MAP_ENTRY_SIZE) as u64;
            #[cfg(unix)]
            {
                use std::os::unix::fs::FileExt;
                ecx_file.read_exact_at(&mut entry_buf, file_offset)?;
            }
            #[cfg(windows)]
            {
                // Positional read so concurrent readers of the shared .ecx
                // handle can't interleave seek/read. Mirrors the
                // read_exact_at helper at the bottom of this file.
                read_exact_at(ecx_file, &mut entry_buf, file_offset)?;
            }
            #[cfg(not(any(unix, windows)))]
            {
                compile_error!("Platform not supported: only unix and windows are supported");
            }
            let (_key, _offset, size) = idx_entry_from_bytes(&entry_buf);
            // Match Go's Size.Raw(): tombstone (-1) returns 0, other negatives return abs
            if !size.is_tombstone() {
                total_size += size.0.unsigned_abs() as u64;
            }
            if size.is_deleted() {
                files_deleted += 1;
            } else {
                files += 1;
            }
        }

        Ok((files, files_deleted, total_size))
    }

    /// ScrubIndex verifies index integrity of an EC volume.
    /// Matches Go's `(ev *EcVolume) ScrubIndex()` → `idx.CheckIndexFile()`.
    /// Returns (entry_count, errors).
    /// Snapshot for `scrub_index`, so the index walk can run with the store
    /// lock released. Same rationale as `checksum_scrub_plan`.
    pub fn scrub_index_plan(&self) -> EcIndexScrubPlan {
        let ecx_path = self.ecx_file_name();
        // Opened under the guard, for the same reason the checksum plan
        // duplicates its shard handles. A fresh open rather than a clone of the
        // cached handle: `check_index_file` seeks, and `dup` would share the
        // cursor the shared handle is read from elsewhere.
        let ecx_handle = File::open(&ecx_path);
        EcIndexScrubPlan {
            volume_id: self.volume_id,
            has_ecx_file: self.ecx_file.is_some(),
            ecx_path,
            ecx_file_size: self.ecx_file_size,
            version: self.version,
            ecx_handle,
        }
    }

    /// Convenience wrapper preserving the original call shape. Callers holding
    /// the store lock MUST use `scrub_index_plan()` + `run()` instead.
    pub fn scrub_index(&self) -> (u64, Vec<String>) {
        self.scrub_index_plan().run()
    }

    /// Snapshot for `scrub_local`, so the needle walk can run with the store
    /// lock released. Same rationale as `checksum_scrub_plan`: this reads every
    /// local needle's bytes, which is GB-scale on a real volume.
    ///
    /// The shard vector keeps its SLOT structure — index is the shard id, gaps
    /// are the shards this node does not hold. `scrub_local` reads that
    /// distinction to decide whether a needle can be reassembled locally at
    /// all, so compacting it would silently change which needles get verified.
    pub fn scrub_local_plan(&self) -> EcLocalScrubPlan {
        EcLocalScrubPlan::for_volumes(&[self]).expect("a single runtime is never an empty slice")
    }

    /// ScrubLocal verifies each needle against the LOCAL shards only; it cannot
    /// CRC-check a needle whose intervals span shards held on other servers.
    /// Mirrors Go's EcVolume.ScrubLocal. Returns (rows walked, broken shards, errors).
    ///
    /// Convenience wrapper preserving the original call shape. Callers that
    /// hold the store lock MUST use `scrub_local_plan()` + `run()` instead.
    pub fn scrub_local(
        &self,
    ) -> (
        u64,
        Vec<crate::pb::volume_server_pb::EcShardInfo>,
        Vec<String>,
    ) {
        self.scrub_local_plan().run()
    }

    // ---- Deletion ----

    /// Write `TOMBSTONE_FILE_SIZE` over the Size field of an existing .ecx
    /// entry, matching Go's `MarkNeedleDeleted`. Only used by the offline
    /// `rebuild_ecx_from_journal` path — the runtime delete path does not
    /// touch .ecx because the index is treated as an immutable sorted
    /// (id, offset, size) table. Returns `false` if the needle is not in
    /// the index (ignored by callers) and an error on IO failure.
    fn tombstone_ecx_entry(&self, needle_id: NeedleId) -> io::Result<bool> {
        let ecx_file = self.ecx_file.as_ref().ok_or_else(|| {
            io::Error::other(format!(
                "ec volume {} has no open .ecx file (closed or corrupt)",
                self.volume_id.0
            ))
        })?;

        let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
        if entry_count == 0 {
            return Ok(false);
        }

        let mut lo: usize = 0;
        let mut hi: usize = entry_count;
        let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
        while lo < hi {
            let mid = lo + (hi - lo) / 2;
            let file_offset = (mid * NEEDLE_MAP_ENTRY_SIZE) as u64;
            #[cfg(unix)]
            {
                use std::os::unix::fs::FileExt;
                ecx_file.read_exact_at(&mut entry_buf, file_offset)?;
            }
            #[cfg(windows)]
            {
                // Positional read so concurrent readers of the shared .ecx
                // handle can't interleave seek/read. Mirrors the
                // read_exact_at helper at the bottom of this file.
                read_exact_at(ecx_file, &mut entry_buf, file_offset)?;
            }
            #[cfg(not(any(unix, windows)))]
            {
                compile_error!("Platform not supported: only unix and windows are supported");
            }
            let (key, _offset, _old_size) = idx_entry_from_bytes(&entry_buf);
            if key == needle_id {
                let size_offset = file_offset + NEEDLE_ID_SIZE as u64 + OFFSET_SIZE as u64;
                let mut size_buf = [0u8; SIZE_SIZE];
                TOMBSTONE_FILE_SIZE.to_bytes(&mut size_buf);
                #[cfg(unix)]
                {
                    use std::os::unix::fs::FileExt;
                    ecx_file.write_all_at(&size_buf, size_offset)?;
                }
                #[cfg(windows)]
                {
                    // Positional write so concurrent readers of the shared
                    // .ecx handle can't observe a moved cursor. Mirrors the
                    // read_exact_at helper at the bottom of this file, with
                    // seek_write in place of seek_read.
                    use std::os::windows::fs::FileExt;
                    let mut written = 0;
                    let mut at = size_offset;
                    while written < size_buf.len() {
                        let n = ecx_file.seek_write(&size_buf[written..], at)?;
                        if n == 0 {
                            return Err(io::Error::new(
                                io::ErrorKind::WriteZero,
                                "seek_write wrote nothing",
                            ));
                        }
                        written += n;
                        at += n as u64;
                    }
                }
                #[cfg(not(any(unix, windows)))]
                {
                    compile_error!("Platform not supported: only unix and windows are supported");
                }
                return Ok(true);
            } else if key < needle_id {
                lo = mid + 1;
            } else {
                hi = mid;
            }
        }
        Ok(false)
    }

    /// Replay .ecj journal entries into .ecx: for each needle id in .ecj,
    /// overwrite its .ecx size field with a tombstone, then remove the
    /// journal file. Mirrors Go's `RebuildEcxFile`, which is invoked from
    /// specific decode / rebuild gRPC handlers — it is intentionally
    /// **not** called on volume load (runtime reads consult
    /// `deleted_needles` instead). The rebuild is atomic with respect to
    /// the journal: if any individual write fails the .ecj file is left
    /// in place and the error is propagated so tombstones are not lost.
    #[expect(dead_code, reason = "no caller yet; see the doc comment")]
    fn rebuild_ecx_from_journal(&mut self) -> io::Result<()> {
        let ecj_path = self.ecj_file_name();
        if !std::path::Path::new(&ecj_path).exists() {
            return Ok(());
        }

        let data = fs::read(&ecj_path)?;
        if data.is_empty() {
            return Ok(());
        }

        let count = data.len() / NEEDLE_ID_SIZE;
        for i in 0..count {
            let start = i * NEEDLE_ID_SIZE;
            if start + NEEDLE_ID_SIZE > data.len() {
                break;
            }
            let needle_id = NeedleId::from_bytes(&data[start..start + NEEDLE_ID_SIZE]);
            // A needle that never made it into .ecx is fine (e.g. the
            // delete raced against encode). Any other IO error aborts the
            // rebuild so the journal survives to be retried later.
            self.tombstone_ecx_entry(needle_id)?;
        }

        // Durably flush the newly-written .ecx tombstones before dropping
        // the journal: the writes went through write_all_at and may still
        // be in page cache.
        if let Some(ref ecx_file) = self.ecx_file {
            ecx_file.sync_all()?;
        }

        // Fold successful — drop and recreate the journal, clear the
        // in-memory deleted set (all of its contents are now materialized
        // in .ecx), and reset the cached size.
        fs::remove_file(&ecj_path)?;
        self.ecj_file = Some(open_ecj_append(&ecj_path)?);
        self.ecj_file_size = 0;
        if let Ok(mut set) = self.deleted_needles.write() {
            set.clear();
        }

        Ok(())
    }

    // ---- Deletion journal ----

    /// Record a needle delete: append the id to the .ecj deletion journal
    /// and insert it into the in-memory deleted set. `.ecx` is not touched
    /// at runtime — it is a sealed sorted (id, offset, size) index and
    /// runtime deletion state lives exclusively in .ecj + `deleted_needles`.
    /// A lookup via `find_needle_from_ecx` masks the id out by returning
    /// `TOMBSTONE_FILE_SIZE` on a subsequent read.
    ///
    /// The .ecj append is the durable commit point. On any failure the
    /// file is truncated back to the pre-append length so the on-disk
    /// journal and in-memory state cannot drift. Only after the sync
    /// succeeds is the id published into the set, so a failure leaves
    /// the delete invisible to readers.
    pub fn journal_delete(&mut self, needle_id: NeedleId) -> io::Result<()> {
        // Look the needle up read-only. Missing is a silent no-op; a
        // pre-existing .ecx tombstone (from a prior decode/rebuild) is
        // mirrored into the in-memory set so delete_count stays accurate
        // without needing to walk .ecx on every heartbeat.
        match self.find_needle_from_ecx_raw(needle_id)? {
            None => return Ok(()),
            Some((_, size)) if size.is_deleted() => {
                if let Ok(mut set) = self.deleted_needles.write() {
                    set.insert(needle_id);
                }
                return Ok(());
            }
            Some(_) => {}
        }

        // Idempotent fast path for repeat deletes — avoids the journal
        // append entirely so the derived delete_count stays stable.
        if self.is_needle_deleted(needle_id) {
            return Ok(());
        }

        let mut buf = [0u8; NEEDLE_ID_SIZE];
        needle_id.to_bytes(&mut buf);
        self.append_journal(&buf)?;
        if let Ok(mut set) = self.deleted_needles.write() {
            set.insert(needle_id);
        }
        Ok(())
    }

    /// Append whole records to `.ecj` and sync them. On any failure the file is
    /// truncated back to the pre-append length so the on-disk journal and
    /// `deleted_needles` cannot drift.
    fn append_journal(&mut self, records: &[u8]) -> io::Result<()> {
        let prev_ecj_size = self.ecj_file_size;
        let append_result: io::Result<()> = {
            let ecj_file = self
                .ecj_file
                .as_mut()
                .ok_or_else(|| io::Error::other("ecj file not open"))?;
            ecj_file
                .write_all(records)
                .and_then(|_| ecj_file.sync_all())
        };

        match append_result {
            Ok(()) => {
                self.ecj_file_size += records.len() as i64;
                Ok(())
            }
            Err(e) => {
                // write_all may have extended the file on disk before
                // sync_all failed; truncate back to the known-good size so
                // the on-disk journal never drifts past `deleted_needles`.
                // Uses its own write handle: on Windows the append handle
                // lacks FILE_WRITE_DATA, so set_len through it fails with
                // ERROR_ACCESS_DENIED and the rollback would silently not
                // happen.
                let ecj_path = self.ecj_file_name();
                let rollback = open_volume_file(OpenOptions::new().write(true), &ecj_path)
                    .and_then(|f| f.set_len(prev_ecj_size as u64).and_then(|_| f.sync_all()));
                // Bumped on attempt, not success: a failed rollback leaves the
                // bytes in place and a decode re-reading for the epoch change
                // simply sees them — harmless — while a successful one must
                // never go unnoticed by an unlocked journal read.
                ECJ_ROLLBACK_EPOCH.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
                if let Err(trunc_err) = rollback {
                    tracing::error!(
                        volume_id = self.volume_id.0,
                        truncate_error = %trunc_err,
                        "failed to truncate ecj after append failure"
                    );
                }
                Err(e)
            }
        }
    }

    /// Internal: binary search .ecx without masking by `deleted_needles`.
    /// Used by `journal_delete` so a repeat delete can still see the raw
    /// pre-existing .ecx tombstone from a prior rebuild.
    fn find_needle_from_ecx_raw(&self, needle_id: NeedleId) -> io::Result<Option<(Offset, Size)>> {
        let ecx_file = self
            .ecx_file
            .as_ref()
            .ok_or_else(|| io::Error::other("ecx file not open"))?;
        let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
        if entry_count == 0 {
            return Ok(None);
        }
        let mut lo: usize = 0;
        let mut hi: usize = entry_count;
        let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
        while lo < hi {
            let mid = lo + (hi - lo) / 2;
            let file_offset = (mid * NEEDLE_MAP_ENTRY_SIZE) as u64;
            #[cfg(unix)]
            {
                use std::os::unix::fs::FileExt;
                ecx_file.read_exact_at(&mut entry_buf, file_offset)?;
            }
            #[cfg(windows)]
            {
                // Positional read so concurrent find_needle_from_ecx_raw calls
                // on the shared .ecx handle don't interleave seek/read and
                // corrupt each other's binary search. Mirrors the
                // read_exact_at helper at the bottom of this file.
                read_exact_at(ecx_file, &mut entry_buf, file_offset)?;
            }
            #[cfg(not(any(unix, windows)))]
            {
                compile_error!("Platform not supported: only unix and windows are supported");
            }
            let (key, offset, size) = idx_entry_from_bytes(&entry_buf);
            if key == needle_id {
                return Ok(Some((offset, size)));
            } else if key < needle_id {
                lo = mid + 1;
            } else {
                hi = mid;
            }
        }
        Ok(None)
    }

    /// Append a deleted needle ID to the .ecj journal, validating the cookie first.
    /// Matches Go's DeleteEcShardNeedle which validates cookie before journaling.
    /// A cookie of 0 means skip cookie check (e.g., orphan cleanup).
    pub fn journal_delete_with_cookie(
        &mut self,
        needle_id: NeedleId,
        cookie: crate::storage::types::Cookie,
    ) -> io::Result<()> {
        // cookie == 0 indicates SkipCookieCheck was requested
        if cookie.0 != 0 {
            let (offset, size) = match self.find_needle_from_ecx(needle_id)? {
                Some((o, s)) => (o, s),
                None => return self.journal_delete(needle_id),
            };
            if size.is_deleted() || offset.is_zero() {
                return self.journal_delete(needle_id);
            }
            let actual_offset = offset.to_actual_offset();
            let intervals = self.locate_ec_shard_needle_interval(actual_offset, size);
            if intervals.is_empty() {
                return Err(io::Error::new(
                    io::ErrorKind::InvalidData,
                    format!("cannot verify cookie for needle {}", needle_id.0),
                ));
            }
            let (shard_id, shard_offset) = self.interval_to_shard_id_and_offset(&intervals[0]);
            let shard = self
                .shards
                .get(shard_id as usize)
                .and_then(|s| s.as_ref())
                .ok_or_else(|| {
                    io::Error::new(
                        io::ErrorKind::InvalidData,
                        format!("cannot verify cookie: shard {} not local", shard_id),
                    )
                })?;
            // Retry short reads, but fail closed on EOF.
            let mut header_buf = [0u8; 4];
            let mut filled = 0usize;
            while filled < header_buf.len() {
                match shard.read_at(
                    &mut header_buf[filled..],
                    shard_offset as u64 + filled as u64,
                ) {
                    Ok(0) => break,
                    Ok(n) => filled += n,
                    Err(e) => {
                        return Err(io::Error::new(
                            io::ErrorKind::InvalidData,
                            format!("cannot verify cookie: {}", e),
                        ));
                    }
                }
            }
            if filled != header_buf.len() {
                return Err(io::Error::new(
                    io::ErrorKind::InvalidData,
                    "cannot verify cookie: incomplete header",
                ));
            }
            let needle_cookie = crate::storage::types::Cookie(u32::from_be_bytes(header_buf));
            if needle_cookie != cookie {
                return Err(io::Error::new(
                    io::ErrorKind::InvalidData,
                    format!("unexpected cookie {:x}", cookie.0),
                ));
            }
        }
        self.journal_delete(needle_id)
    }

    /// Read all deleted needle IDs from the .ecj journal.
    pub fn read_deleted_needles(&self) -> io::Result<Vec<NeedleId>> {
        let ecj_path = self.ecj_file_name();
        if !std::path::Path::new(&ecj_path).exists() {
            return Ok(Vec::new());
        }

        let data = fs::read(&ecj_path)?;
        let count = data.len() / NEEDLE_ID_SIZE;
        let mut needles = Vec::with_capacity(count);
        for i in 0..count {
            let start = i * NEEDLE_ID_SIZE;
            let id = NeedleId::from_bytes(&data[start..start + NEEDLE_ID_SIZE]);
            needles.push(id);
        }
        Ok(needles)
    }

    /// Fold a peer's deletion journal into this mounted volume: append only
    /// the ids not already deleted here, in one write and one fsync, then
    /// publish them into the in-memory set — the same commit order as
    /// `journal_delete`. `&mut self` serializes it with deletes under the store
    /// lock, and the live handle is appended to in place, so no delete can land
    /// in an orphaned file. Returns how many ids were added.
    pub fn merge_journal(&mut self, ids: &HashSet<NeedleId>) -> io::Result<usize> {
        let delta = super::ecj_merge::ecj_delta(ids, |id| self.is_needle_deleted(*id));
        if delta.is_empty() {
            return Ok(0);
        }
        self.append_journal(&super::ecj_merge::encode_ecj_ids(&delta))?;
        if let Ok(mut set) = self.deleted_needles.write() {
            set.extend(delta.iter().copied());
        }
        Ok(delta.len())
    }

    /// Marks ids deleted in memory only, for runtimes sharing a journal
    /// another runtime already appended them to.
    pub fn publish_merged_ids(&self, ids: &HashSet<NeedleId>) {
        if let Ok(mut set) = self.deleted_needles.write() {
            set.extend(ids.iter().copied());
        }
    }

    // ---- Lifecycle ----

    pub fn close(&mut self) {
        for shard in &mut self.shards {
            if let Some(s) = shard {
                s.close();
            }
            *shard = None;
        }
        // Sync .ecx before closing to flush in-place deletion marks (matches Go's ev.ecxFile.Sync())
        if let Some(ref ecx_file) = self.ecx_file {
            let _ = ecx_file.sync_all();
        }
        self.ecx_file = None;
        self.ecj_file = None;
        self.ecj_hold = None;
    }

    pub fn destroy(&mut self) {
        for shard in &mut self.shards {
            if let Some(s) = shard {
                s.destroy();
            }
            *shard = None;
        }
        // Remove .ecx/.ecj/.vif from ecx_actual_dir (where they were found)
        // Go's Destroy() removes .ecx, .ecj, and .vif files.
        let actual_base = crate::storage::volume::volume_file_name(
            &self.ecx_actual_dir,
            &self.collection,
            self.volume_id,
        );
        let _ = fs::remove_file(format!("{}.ecx", actual_base));
        let _ = fs::remove_file(format!("{}.ecj", actual_base));
        let _ = fs::remove_file(format!("{}{}", actual_base, ECJ_COMPACT_TMP_EXT));
        let _ = fs::remove_file(format!("{}.vif", actual_base));
        // Also sweep the originally-configured idx dir in case stale files
        // exist there (ecx_file_name() / ecj_file_name() now resolve from
        // ecx_actual_dir, so we have to build the idx-dir paths explicitly).
        if self.ecx_actual_dir != self.dir_idx {
            let idx_base = crate::storage::volume::volume_file_name(
                &self.dir_idx,
                &self.collection,
                self.volume_id,
            );
            let _ = fs::remove_file(format!("{}.ecx", idx_base));
            let _ = fs::remove_file(format!("{}.ecj", idx_base));
            let _ = fs::remove_file(format!("{}{}", idx_base, ECJ_COMPACT_TMP_EXT));
            let _ = fs::remove_file(format!("{}.vif", idx_base));
        }
        if self.ecx_actual_dir != self.dir && self.dir_idx != self.dir {
            let data_base = crate::storage::volume::volume_file_name(
                &self.dir,
                &self.collection,
                self.volume_id,
            );
            let _ = fs::remove_file(format!("{}.ecx", data_base));
            let _ = fs::remove_file(format!("{}.ecj", data_base));
            let _ = fs::remove_file(format!("{}{}", data_base, ECJ_COMPACT_TMP_EXT));
            let _ = fs::remove_file(format!("{}.vif", data_base));
        }
        // Go's Destroy() also removes bitrot checksum sidecars so a later
        // volume-id reuse cannot load stale protection, and so
        // collection.delete does not leave orphaned <base>.ecsum files.
        // ecx_actual_dir is always one of these two dirs.
        let _ =
            crate::storage::erasure_coding::ec_bitrot::remove_bitrot_sidecars(&self.base_name());
        if self.dir_idx != self.dir {
            let idx_base = crate::storage::volume::volume_file_name(
                &self.dir_idx,
                &self.collection,
                self.volume_id,
            );
            let _ = crate::storage::erasure_coding::ec_bitrot::remove_bitrot_sidecars(&idx_base);
        }
        self.ecx_file = None;
        self.ecj_file = None;
        self.ecj_hold = None;
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::TempDir;

    /// Go's NewEcVolume fails with os.ErrNotExist when neither directory has an
    /// `.ecx`. Mounting anyway advertises shards every read then fails on with
    /// "ecx file not open", and zeroes the size `add_shard`'s 0-byte guard needs.
    #[test]
    fn test_new_without_ecx_is_not_found() {
        let data = TempDir::new().unwrap();
        let idx = TempDir::new().unwrap();
        let (dir, dir_idx) = (data.path().to_str().unwrap(), idx.path().to_str().unwrap());
        std::fs::write(format!("{}/7.ec00", dir), b"shard").unwrap();

        for idx_dir in [dir, dir_idx] {
            let err = EcVolume::new(dir, idx_dir, "", VolumeId(7))
                .err()
                .expect("an EC volume without an .ecx must not mount");
            assert_eq!(err.kind(), io::ErrorKind::NotFound, "{}", err);
        }
    }

    /// A 0-byte `.ecx` stub left by an interrupted copy yields to a non-empty
    /// copy in the other directory, whichever side the stub is on.
    #[test]
    fn test_new_prefers_non_empty_ecx_over_zero_byte_stub() {
        for stub_in_idx_dir in [true, false] {
            let data = TempDir::new().unwrap();
            let idx = TempDir::new().unwrap();
            let (dir, dir_idx) = (data.path().to_str().unwrap(), idx.path().to_str().unwrap());
            let (stub_dir, valid_dir) = if stub_in_idx_dir {
                (dir_idx, dir)
            } else {
                (dir, dir_idx)
            };
            std::fs::write(format!("{}/7.ecx", stub_dir), b"").unwrap();
            std::fs::write(
                format!("{}/7.ecx", valid_dir),
                vec![0u8; NEEDLE_MAP_ENTRY_SIZE],
            )
            .unwrap();

            let vol = EcVolume::new(dir, dir_idx, "", VolumeId(7)).unwrap();
            assert_eq!(vol.ecx_actual_dir(), valid_dir);
            assert_eq!(vol.ecx_file_size, NEEDLE_MAP_ENTRY_SIZE as i64);
        }
    }

    /// With no other copy a 0-byte `.ecx` is a legitimate empty index (a volume
    /// whose needles were all deleted before encoding) and still mounts, as in Go.
    #[test]
    fn test_new_accepts_lone_zero_byte_ecx_as_empty_index() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        std::fs::write(format!("{}/7.ecx", dir), b"").unwrap();

        let vol = EcVolume::new(dir, dir, "", VolumeId(7)).unwrap();
        assert_eq!(vol.ecx_file_size, 0);
        assert!(!is_usable_ecx_file(&vol.ecx_file_name()));
    }

    /// `destroy()` must remove co-located `.ecsum` sidecars (Go Destroy parity).
    /// Without this, `collection.delete` leaves orphaned bitrot files that
    /// inflate EC-health scanners after the shards are gone.
    #[test]
    fn test_destroy_removes_bitrot_sidecar() {
        use crate::storage::needle_map::NeedleMapKind;
        use crate::storage::volume::{Volume, VolumeSpec};

        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let mut v = Volume::new(
            dir,
            dir,
            VolumeId(2074),
            NeedleMapKind::InMemory,
            &VolumeSpec {
                collection: "ec1c",
                ..Default::default()
            },
        )
        .unwrap();
        for i in 1..=3 {
            let data = format!("needle {}", i);
            let mut n = Needle {
                id: NeedleId(i),
                cookie: Cookie(i as u32),
                data: data.as_bytes().to_vec(),
                data_size: data.len() as u32,
                ..Needle::default()
            };
            v.write_needle(&mut n, true, false).unwrap();
        }
        v.sync_to_disk().unwrap();
        v.close();
        crate::storage::erasure_coding::ec_encoder::write_ec_files(
            dir,
            dir,
            "ec1c",
            VolumeId(2074),
            10,
            4,
        )
        .unwrap();

        let base = crate::storage::volume::volume_file_name(dir, "ec1c", VolumeId(2074));
        let ecsum = format!("{}.ecsum", base);
        assert!(
            std::path::Path::new(&ecsum).exists(),
            "precondition: encode must write generation-0 .ecsum"
        );
        assert!(
            std::path::Path::new(&format!("{}.ec00", base)).exists(),
            "precondition: encode must write shards"
        );

        let mut vol = EcVolume::new(dir, dir, "ec1c", VolumeId(2074)).unwrap();
        // Mount at least one local shard so destroy's shard loop has work.
        vol.add_shard(EcVolumeShard::new(dir, "ec1c", VolumeId(2074), 0))
            .unwrap();
        vol.destroy();

        assert!(
            !std::path::Path::new(&format!("{}.ec00", base)).exists(),
            "destroy should remove shards"
        );
        assert!(
            !std::path::Path::new(&ecsum).exists(),
            "destroy should remove co-located .ecsum (Go Destroy parity)"
        );
        assert!(
            !std::path::Path::new(&format!("{}.ecx", base)).exists(),
            "destroy should remove .ecx"
        );
    }

    /// Mounting an EC volume loads and validates its generation-0 `.ecsum`
    /// sidecar, so `bitrot_protection()` reports `On` with the parsed manifest.
    #[test]
    fn test_mount_loads_bitrot_sidecar() {
        use crate::storage::erasure_coding::ec_bitrot::BitrotStatus;
        use crate::storage::needle_map::NeedleMapKind;
        use crate::storage::volume::{Volume, VolumeSpec};

        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let mut v = Volume::new(
            dir,
            dir,
            VolumeId(1),
            NeedleMapKind::InMemory,
            &VolumeSpec::default(),
        )
        .unwrap();
        for i in 1..=5 {
            let data = format!("test data for needle {}", i);
            let mut n = Needle {
                id: NeedleId(i),
                cookie: Cookie(i as u32),
                data: data.as_bytes().to_vec(),
                data_size: data.len() as u32,
                ..Needle::default()
            };
            v.write_needle(&mut n, true, false).unwrap();
        }
        v.sync_to_disk().unwrap();
        v.close();
        crate::storage::erasure_coding::ec_encoder::write_ec_files(
            dir,
            dir,
            "",
            VolumeId(1),
            10,
            4,
        )
        .unwrap();

        let vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        assert!(
            vol.bitrot.is_some(),
            "mount should load the generation-0 .ecsum sidecar"
        );
        let (prot, status) = vol.bitrot_protection();
        assert_eq!(status, BitrotStatus::On);
        assert_eq!(prot.unwrap().shards.len(), 14);
    }

    /// REGRESSION: the scrub plans must be SELF-CONTAINED, so the handler can
    /// drop the store lock before the scan runs — see `EcChecksumScrubPlan`.
    ///
    /// The volume is DROPPED before the plans run, and the plans are moved to
    /// another thread. A plan that borrowed from `EcVolume` could do neither, so
    /// this stops COMPILING if the snapshot ever regresses to a borrow.
    #[test]
    fn test_scrub_plans_are_self_contained_and_match_direct_call() {
        use crate::storage::needle_map::NeedleMapKind;
        use crate::storage::volume::{Volume, VolumeSpec};

        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let mut v = Volume::new(
            dir,
            dir,
            VolumeId(1),
            NeedleMapKind::InMemory,
            &VolumeSpec::default(),
        )
        .unwrap();
        for i in 1..=8 {
            let data = format!("test data for needle {} with a bit more length", i);
            let mut n = Needle {
                id: NeedleId(i),
                cookie: Cookie(i as u32),
                data: data.as_bytes().to_vec(),
                data_size: data.len() as u32,
                ..Needle::default()
            };
            v.write_needle(&mut n, true, false).unwrap();
        }
        v.sync_to_disk().unwrap();
        v.close();
        crate::storage::erasure_coding::ec_encoder::write_ec_files(
            dir,
            dir,
            "",
            VolumeId(1),
            10,
            4,
        )
        .unwrap();

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        for id in 0..14u8 {
            vol.add_shard(EcVolumeShard::new(dir, "", VolumeId(1), id))
                .unwrap();
        }

        // Baseline via the original call shape, with the volume still alive.
        let direct_checksum = vol.checksum_scrub();
        let direct_index = vol.scrub_index();
        let direct_local = vol.scrub_local();
        assert!(
            direct_checksum.0 > 0 && direct_local.0 > 0,
            "fixture scanned nothing, so the equalities below would be vacuous"
        );

        let checksum_plan = vol.checksum_scrub_plan();
        let index_plan = vol.scrub_index_plan();
        let local_plan = vol.scrub_local_plan();

        // The lock (and here the whole volume) is gone before the scan runs.
        drop(vol);

        let (from_plan_checksum, from_plan_index, from_plan_local) =
            std::thread::spawn(move || (checksum_plan.run(), index_plan.run(), local_plan.run()))
                .join()
                .expect("scrub plans must be runnable off the owning thread");

        assert_eq!(
            from_plan_checksum, direct_checksum,
            "checksum scrub result changed when run from a released-lock plan"
        );
        assert_eq!(
            from_plan_index, direct_index,
            "index scrub result changed when run from a released-lock plan"
        );
        assert_eq!(
            from_plan_local, direct_local,
            "local scrub result changed when run from a released-lock plan"
        );
    }

    /// REGRESSION: a malformed `.ecx` row must not abort the scrub TASK.
    ///
    /// `EcLocalScrubPlan::run()` skips only the -1 tombstone, matching Go's
    /// `ScrubLocal`, so any OTHER negative size reaches the reassembly buffer
    /// with a negative `get_actual_size()`. Go pays nothing for that (it
    /// appends to a nil slice); Rust sizes a per-needle `Vec` from it, and
    /// `Vec::with_capacity(negative as usize)` aborts the process. Now that the
    /// plan runs under `spawn_blocking`, that abort comes back as a JoinError
    /// and would take the whole node-wide scrub RPC down with every result
    /// already collected. The row must be REPORTED, as Go reports it.
    #[test]
    fn test_local_scrub_plan_reports_negative_size_ecx_row() {
        use crate::storage::needle_map::NeedleMapKind;
        use crate::storage::volume::{Volume, VolumeSpec};

        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let mut v = Volume::new(
            dir,
            dir,
            VolumeId(1),
            NeedleMapKind::InMemory,
            &VolumeSpec::default(),
        )
        .unwrap();
        for i in 1..=8 {
            let data = format!("test data for needle {} with a bit more length", i);
            let mut n = Needle {
                id: NeedleId(i),
                cookie: Cookie(i as u32),
                data: data.as_bytes().to_vec(),
                data_size: data.len() as u32,
                ..Needle::default()
            };
            v.write_needle(&mut n, true, false).unwrap();
        }
        v.sync_to_disk().unwrap();
        v.close();
        crate::storage::erasure_coding::ec_encoder::write_ec_files(
            dir,
            dir,
            "",
            VolumeId(1),
            10,
            4,
        )
        .unwrap();

        // Rewrite the first .ecx row's size as -1000: a negative that is NOT
        // the -1 tombstone the walk skips. A scrub is what you point at an
        // index you already suspect, so an arbitrary i32 in the size field is
        // in-scope input, whatever wrote it.
        let ecx = format!(
            "{}.ecx",
            crate::storage::volume::volume_file_name(dir, "", VolumeId(1))
        );
        let mut raw = std::fs::read(&ecx).unwrap();
        assert!(
            raw.len() >= NEEDLE_MAP_ENTRY_SIZE,
            "fixture must write at least one .ecx row"
        );
        let (key, offset, _) = idx_entry_from_bytes(&raw[..NEEDLE_MAP_ENTRY_SIZE]);
        idx_entry_to_bytes(&mut raw[..NEEDLE_MAP_ENTRY_SIZE], key, offset, Size(-1000));
        std::fs::write(&ecx, &raw).unwrap();
        assert!(
            get_actual_size(Size(-1000), Version::current()) < 0,
            "precondition: the row must drive get_actual_size negative"
        );

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        for id in 0..14u8 {
            vol.add_shard(EcVolumeShard::new(dir, "", VolumeId(1), id))
                .unwrap();
        }
        let plan = vol.scrub_local_plan();
        drop(vol);

        // The join IS the assertion: a panic here is the JoinError that used to
        // fail the whole ScrubEcVolume RPC.
        let (_count, _broken, errs) = std::thread::spawn(move || plan.run())
            .join()
            .expect("a malformed .ecx row must not abort the scrub task");

        assert!(
            errs.iter()
                .any(|e| e.contains(&format!("bytes for needle {}", key.0))),
            "the malformed row must be reported, got {:?}",
            errs
        );
    }

    /// REGRESSION: a scrub running with the store lock RELEASED must not turn a
    /// concurrent, intentional removal into a corruption report — see
    /// `EcChecksumScrubPlan`.
    ///
    /// Deleting every file after the plans are built is the whole test: reads
    /// that resolve a path diverge from the direct call, reads through the
    /// captured descriptors are identical.
    #[test]
    fn test_scrub_plans_survive_files_removed_after_snapshot() {
        use crate::storage::needle_map::NeedleMapKind;
        use crate::storage::volume::{Volume, VolumeSpec};

        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let mut v = Volume::new(
            dir,
            dir,
            VolumeId(1),
            NeedleMapKind::InMemory,
            &VolumeSpec::default(),
        )
        .unwrap();
        for i in 1..=8 {
            let data = format!("test data for needle {} with a bit more length", i);
            let mut n = Needle {
                id: NeedleId(i),
                cookie: Cookie(i as u32),
                data: data.as_bytes().to_vec(),
                data_size: data.len() as u32,
                ..Needle::default()
            };
            v.write_needle(&mut n, true, false).unwrap();
        }
        v.sync_to_disk().unwrap();
        v.close();
        crate::storage::erasure_coding::ec_encoder::write_ec_files(
            dir,
            dir,
            "",
            VolumeId(1),
            10,
            4,
        )
        .unwrap();

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        for id in 0..14u8 {
            vol.add_shard(EcVolumeShard::new(dir, "", VolumeId(1), id))
                .unwrap();
        }

        // Baseline with every file present and the volume still mounted.
        let direct_checksum = vol.checksum_scrub();
        let direct_index = vol.scrub_index();
        let direct_local = vol.scrub_local();
        assert!(
            direct_checksum.0 > 0 && direct_local.0 > 0,
            "fixture scanned nothing, so the equalities below would be vacuous"
        );
        assert!(
            direct_checksum.2.is_empty() && direct_index.1.is_empty() && direct_local.2.is_empty(),
            "fixture is not clean, so a false error could not be told apart: {:?} {:?} {:?}",
            direct_checksum.2,
            direct_index.1,
            direct_local.2
        );

        let checksum_plan = vol.checksum_scrub_plan();
        let index_plan = vol.scrub_index_plan();
        let local_plan = vol.scrub_local_plan();

        // The teardown a store writer would perform, after the plans exist.
        drop(vol);
        let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(1));
        for id in 0..14u8 {
            std::fs::remove_file(format!("{}.ec{:02}", base, id)).unwrap();
        }
        std::fs::remove_file(format!("{}.ecx", base)).unwrap();
        assert!(
            !std::path::Path::new(&format!("{}.ec00", base)).exists(),
            "the removal under test did not happen"
        );

        assert_eq!(
            checksum_plan.run(),
            direct_checksum,
            "shard files unlinked after the snapshot were reported as corruption"
        );
        assert_eq!(
            index_plan.run(),
            direct_index,
            "the .ecx unlinked after the snapshot was reported as an index error"
        );
        assert_eq!(
            local_plan.run(),
            direct_local,
            "files unlinked after the snapshot were reported as local-scrub errors"
        );
    }

    /// CHECKSUM scrub verifies clean shards against the sidecar and flags a shard
    /// whose bytes are corrupted after encode.
    #[test]
    fn test_checksum_scrub_clean_and_detects_corruption() {
        use crate::storage::needle_map::NeedleMapKind;
        use crate::storage::volume::{Volume, VolumeSpec};

        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let mut v = Volume::new(
            dir,
            dir,
            VolumeId(1),
            NeedleMapKind::InMemory,
            &VolumeSpec::default(),
        )
        .unwrap();
        for i in 1..=8 {
            let data = format!("test data for needle {} with a bit more length", i);
            let mut n = Needle {
                id: NeedleId(i),
                cookie: Cookie(i as u32),
                data: data.as_bytes().to_vec(),
                data_size: data.len() as u32,
                ..Needle::default()
            };
            v.write_needle(&mut n, true, false).unwrap();
        }
        v.sync_to_disk().unwrap();
        v.close();
        crate::storage::erasure_coding::ec_encoder::write_ec_files(
            dir,
            dir,
            "",
            VolumeId(1),
            10,
            4,
        )
        .unwrap();

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        for id in 0..14u8 {
            vol.add_shard(EcVolumeShard::new(dir, "", VolumeId(1), id))
                .unwrap();
        }

        // Clean scrub: no mismatches, no errors, blocks scanned > 0.
        let (scanned, broken, errs) = vol.checksum_scrub();
        assert!(errs.is_empty(), "unexpected scrub errors: {:?}", errs);
        assert!(broken.is_empty(), "unexpected mismatches: {:?}", broken);
        assert!(scanned > 0, "scrub should scan at least one block");

        // Corrupt one byte of shard 3 on disk; re-scrub must report shard 3.
        let shard3 = format!("{}/1.ec03", dir);
        let mut bytes = std::fs::read(&shard3).unwrap();
        assert!(!bytes.is_empty());
        bytes[0] ^= 0xFF;
        std::fs::write(&shard3, &bytes).unwrap();

        let (_, broken2, _) = vol.checksum_scrub();
        assert!(
            broken2.contains(&3),
            "corrupted shard 3 should be flagged, got {:?}",
            broken2
        );
    }

    fn write_ecx_file(
        dir: &str,
        collection: &str,
        vid: VolumeId,
        entries: &[(NeedleId, Offset, Size)],
    ) {
        let base = crate::storage::volume::volume_file_name(dir, collection, vid);
        let ecx_path = format!("{}.ecx", base);
        let mut file = File::create(&ecx_path).unwrap();

        // Write sorted entries
        for &(key, offset, size) in entries {
            let mut buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
            idx_entry_to_bytes(&mut buf, key, offset, size);
            file.write_all(&buf).unwrap();
        }
    }

    #[test]
    fn test_ec_volume_find_needle() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();

        // Write sorted ecx entries
        let entries = vec![
            (NeedleId(1), Offset::from_actual_offset(8), Size(100)),
            (NeedleId(5), Offset::from_actual_offset(200), Size(200)),
            (NeedleId(10), Offset::from_actual_offset(500), Size(300)),
        ];
        write_ecx_file(dir, "", VolumeId(1), &entries);

        let vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();

        // Found
        let result = vol.find_needle_from_ecx(NeedleId(5)).unwrap();
        assert!(result.is_some());
        let (offset, size) = result.unwrap();
        assert_eq!(offset.to_actual_offset(), 200);
        assert_eq!(size, Size(200));

        // Not found
        let result = vol.find_needle_from_ecx(NeedleId(7)).unwrap();
        assert!(result.is_none());
    }

    #[test]
    fn test_ec_volume_journal() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();

        // .ecj append is gated on a live->tombstone transition in .ecx, so
        // the fixture must contain the needles we are about to delete.
        let entries = vec![
            (NeedleId(10), Offset::from_actual_offset(8), Size(100)),
            (NeedleId(20), Offset::from_actual_offset(200), Size(200)),
        ];
        write_ecx_file(dir, "", VolumeId(1), &entries);

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        let (fc0, dc0) = vol.file_and_delete_count();
        assert_eq!((fc0, dc0), (2, 0));

        vol.journal_delete(NeedleId(10)).unwrap();
        vol.journal_delete(NeedleId(20)).unwrap();

        let deleted = vol.read_deleted_needles().unwrap();
        assert_eq!(deleted, vec![NeedleId(10), NeedleId(20)]);

        let (fc, dc) = vol.file_and_delete_count();
        assert_eq!((fc, dc), (2, 2));

        // Idempotent re-delete must not bump delete_count.
        vol.journal_delete(NeedleId(10)).unwrap();
        // Deleting a missing needle must not bump delete_count either.
        vol.journal_delete(NeedleId(999)).unwrap();
        let (fc, dc) = vol.file_and_delete_count();
        assert_eq!((fc, dc), (2, 2));
    }

    /// A mounted volume merges a peer's journal through its own handle: only
    /// the missing ids are appended, the file keeps its inode, the in-memory
    /// set follows, and a later delete lands in the same live file.
    #[test]
    fn test_merge_journal_appends_missing_ids_in_place() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let entries: Vec<_> = (1..=5)
            .map(|id| {
                (
                    NeedleId(id),
                    Offset::from_actual_offset(8 * id as i64),
                    Size(100),
                )
            })
            .collect();
        write_ecx_file(dir, "", VolumeId(1), &entries);

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        vol.journal_delete(NeedleId(1)).unwrap();
        let ecj_path = vol.ecj_file_name();
        #[cfg(unix)]
        let inode = {
            use std::os::unix::fs::MetadataExt;
            std::fs::metadata(&ecj_path).unwrap().ino()
        };

        let peer: HashSet<NeedleId> = [1, 2, 3].into_iter().map(NeedleId).collect();
        assert_eq!(vol.merge_journal(&peer).unwrap(), 2);
        assert_eq!(
            vol.merge_journal(&peer).unwrap(),
            0,
            "a repeated merge appends nothing"
        );
        for id in 1..=3 {
            assert!(vol.is_needle_deleted(NeedleId(id)), "id {}", id);
        }
        assert_eq!(vol.file_and_delete_count().1, 3);

        vol.journal_delete(NeedleId(4)).unwrap();
        #[cfg(unix)]
        {
            use std::os::unix::fs::MetadataExt;
            assert_eq!(
                std::fs::metadata(&ecj_path).unwrap().ino(),
                inode,
                "the journal must never be replaced under the open handle"
            );
        }
        assert_eq!(
            vol.read_deleted_needles().unwrap(),
            (1..=4).map(NeedleId).collect::<Vec<_>>()
        );
        vol.close();

        let vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        for id in 1..=4 {
            assert!(
                vol.is_needle_deleted(NeedleId(id)),
                "id {} survives remount",
                id
            );
        }
    }

    /// Write a raw `.ecj` containing `ids` repeated `repeats` times, i.e. the
    /// shape the append paths produce when a peer's whole journal is
    /// concatenated onto this one over and over.
    fn write_bloated_ecj(
        dir: &str,
        collection: &str,
        vid: VolumeId,
        ids: &[NeedleId],
        repeats: usize,
    ) {
        let base = crate::storage::volume::volume_file_name(dir, collection, vid);
        let mut one = vec![0u8; ids.len() * NEEDLE_ID_SIZE];
        for (i, id) in ids.iter().enumerate() {
            id.to_bytes(&mut one[i * NEEDLE_ID_SIZE..(i + 1) * NEEDLE_ID_SIZE]);
        }
        let mut f = File::create(format!("{}.ecj", base)).unwrap();
        for _ in 0..repeats {
            f.write_all(&one).unwrap();
        }
        f.sync_all().unwrap();
    }
    /// A journal whose length is not a whole number of entries must not lose
    /// the entries that ARE complete, and must not panic.
    #[test]
    fn test_ecj_with_trailing_partial_entry() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        write_ecx_file(dir, "", VolumeId(40), &[]);

        let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(40));
        let ids: Vec<NeedleId> = (1..=3).map(NeedleId).collect();
        let mut bytes = vec![0u8; ids.len() * NEEDLE_ID_SIZE];
        for (i, id) in ids.iter().enumerate() {
            id.to_bytes(&mut bytes[i * NEEDLE_ID_SIZE..(i + 1) * NEEDLE_ID_SIZE]);
        }
        bytes.extend_from_slice(&[0xAB, 0xCD, 0xEF]); // torn tail
        std::fs::write(format!("{}.ecj", base), &bytes).unwrap();

        let vol = EcVolume::new(dir, dir, "", VolumeId(40)).unwrap();
        assert_eq!(vol.read_deleted_needles().unwrap(), ids);
    }

    /// A torn tail must be truncated at mount, not merely skipped. The handle
    /// is in append mode, so leaving the partial bytes in place would push
    /// every later append out of alignment: the delete taken after the tear
    /// would decode as garbage on the next mount and be silently lost.
    #[test]
    fn test_torn_ecj_tail_is_repaired_so_later_deletes_survive() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        // journal_delete only appends on a live->tombstone transition.
        write_ecx_file(
            dir,
            "",
            VolumeId(42),
            &[(NeedleId(7), Offset::from_actual_offset(8), Size(10))],
        );

        let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(42));
        let ecj_path = format!("{}.ecj", base);
        let ids: Vec<NeedleId> = (1..=3).map(NeedleId).collect();
        let mut bytes = vec![0u8; ids.len() * NEEDLE_ID_SIZE];
        for (i, id) in ids.iter().enumerate() {
            id.to_bytes(&mut bytes[i * NEEDLE_ID_SIZE..(i + 1) * NEEDLE_ID_SIZE]);
        }
        bytes.extend_from_slice(&[0xAB, 0xCD, 0xEF]); // torn tail
        std::fs::write(&ecj_path, &bytes).unwrap();

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(42)).unwrap();

        // The tear is gone from disk, not just ignored in memory.
        assert_eq!(
            std::fs::metadata(&ecj_path).unwrap().len(),
            (ids.len() * NEEDLE_ID_SIZE) as u64,
            "torn tail should have been truncated at mount",
        );

        vol.journal_delete(NeedleId(7)).unwrap();
        drop(vol);

        // The delete taken after the repair must survive a remount.
        let vol2 = EcVolume::new(dir, dir, "", VolumeId(42)).unwrap();
        let deleted = vol2.read_deleted_needles().unwrap();
        assert!(
            deleted.contains(&NeedleId(7)),
            "delete after a torn tail was lost: {:?}",
            deleted,
        );
        assert_eq!(
            deleted.len(),
            ids.len() + 1,
            "misaligned decode: {:?}",
            deleted
        );
    }

    /// A journal spanning several read chunks must load every entry — guards
    /// the chunk-boundary arithmetic in the buffered loader.
    #[test]
    fn test_ecj_spanning_multiple_read_chunks() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        write_ecx_file(dir, "", VolumeId(41), &[]);

        // 2.5 chunks' worth of DISTINCT ids, so nothing is masked by dedup and
        // the total is not a chunk multiple.
        let n = (ECJ_LOAD_CHUNK_BYTES / NEEDLE_ID_SIZE) * 5 / 2;
        let ids: Vec<NeedleId> = (1..=n as u64).map(NeedleId).collect();
        write_bloated_ecj(dir, "", VolumeId(41), &ids, 1);

        let vol = EcVolume::new(dir, dir, "", VolumeId(41)).unwrap();
        let deleted = vol.read_deleted_needles().unwrap();
        assert_eq!(deleted.len(), n, "entries lost across a chunk boundary");
        assert_eq!(deleted, ids);
    }

    /// A journal of 1M records over 100 distinct ids must mount to those 100
    /// ids and be folded down to 100x8 bytes. The production failure: 1.51 TB
    /// of ~100 distinct ids wedging startup.
    #[test]
    fn test_bloated_ecj_is_compacted_on_load() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        write_ecx_file(dir, "", VolumeId(37), &[]);

        // 100 unique ids x 10000 repeats x 8 B = 8 MiB on disk for 800 B of
        // actual information — over both compaction thresholds.
        let ids: Vec<NeedleId> = (1000..1100).map(NeedleId).collect();
        write_bloated_ecj(dir, "", VolumeId(37), &ids, 10_000);

        let ecj_path = format!(
            "{}.ecj",
            crate::storage::volume::volume_file_name(dir, "", VolumeId(37))
        );
        let before = std::fs::metadata(&ecj_path).unwrap().len();
        assert_eq!(before, 100 * 10_000 * NEEDLE_ID_SIZE as u64);

        let vol = EcVolume::new(dir, dir, "", VolumeId(37)).unwrap();

        let deleted = vol.read_deleted_needles().unwrap();
        assert_eq!(deleted, ids, "compaction must preserve the deleted set");

        let after = std::fs::metadata(&ecj_path).unwrap().len();
        assert_eq!(
            after,
            ids.len() as u64 * NEEDLE_ID_SIZE as u64,
            "journal should have been folded down to one entry per id",
        );
        assert!(after < before);
        assert!(!std::path::Path::new(&format!("{}.compact.tmp", ecj_path)).exists());

        drop(vol);
        let vol2 = EcVolume::new(dir, dir, "", VolumeId(37)).unwrap();
        assert_eq!(vol2.read_deleted_needles().unwrap(), ids);
        assert_eq!(std::fs::metadata(&ecj_path).unwrap().len(), after);
    }

    /// Appends must still land in the file after a compaction reopened the
    /// handle — otherwise deletes taken after mount would be lost on restart.
    #[test]
    fn test_journal_delete_after_compaction_persists() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        write_ecx_file(
            dir,
            "",
            VolumeId(38),
            &[(NeedleId(7), Offset::from_actual_offset(8), Size(10))],
        );
        let ids: Vec<NeedleId> = (1000..1100).map(NeedleId).collect();
        write_bloated_ecj(dir, "", VolumeId(38), &ids, 4096);

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(38)).unwrap();
        vol.journal_delete(NeedleId(7)).unwrap();
        drop(vol);

        let vol2 = EcVolume::new(dir, dir, "", VolumeId(38)).unwrap();
        let deleted = vol2.read_deleted_needles().unwrap();
        assert!(
            deleted.contains(&NeedleId(7)),
            "post-compaction append was lost across remount: {:?}",
            &deleted[..deleted.len().min(5)],
        );
        assert_eq!(deleted.len(), ids.len() + 1);
    }

    /// A healthy journal must be left byte-for-byte alone — compaction is for
    /// pathology, not routine churn on every mount.
    #[test]
    fn test_healthy_ecj_is_not_rewritten() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        write_ecx_file(dir, "", VolumeId(39), &[]);

        // Duplicated 3x, but only 2.4 KB — under ECJ_COMPACT_MIN_BYTES.
        let ids: Vec<NeedleId> = (1..=100).map(NeedleId).collect();
        write_bloated_ecj(dir, "", VolumeId(39), &ids, 3);

        let ecj_path = format!(
            "{}.ecj",
            crate::storage::volume::volume_file_name(dir, "", VolumeId(39))
        );
        let before_meta = std::fs::metadata(&ecj_path).unwrap();
        let before_bytes = std::fs::read(&ecj_path).unwrap();
        // Use mtime+inode proxy: metadata len + content equality. A rewrite
        // would change mtime; content equality proves no-op.
        let before_mtime = before_meta.modified().unwrap();

        let vol = EcVolume::new(dir, dir, "", VolumeId(39)).unwrap();

        let (_, delete_count) = vol.file_and_delete_count();
        assert_eq!(delete_count, ids.len() as u64);
        assert_eq!(vol.read_deleted_needles().unwrap().len(), ids.len() * 3);

        let after_bytes = std::fs::read(&ecj_path).unwrap();
        assert_eq!(
            before_bytes, after_bytes,
            "small journal must not be rewritten"
        );
        let after_mtime = std::fs::metadata(&ecj_path).unwrap().modified().unwrap();
        assert_eq!(
            before_mtime, after_mtime,
            "small journal mtime must be unchanged"
        );
    }

    /// A torn tail plus an oversized journal must be repaired and compacted,
    /// with no id lost or invented.
    #[test]
    fn test_torn_tail_plus_bloated_journal_is_repaired_and_compacted() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        write_ecx_file(dir, "", VolumeId(45), &[]);

        let ids: Vec<NeedleId> = (1000..1100).map(NeedleId).collect();
        write_bloated_ecj(dir, "", VolumeId(45), &ids, 4096);
        let ecj_path = format!(
            "{}.ecj",
            crate::storage::volume::volume_file_name(dir, "", VolumeId(45))
        );
        // Append a torn tail.
        {
            use std::io::Write as _;
            let mut f = std::fs::OpenOptions::new()
                .append(true)
                .open(&ecj_path)
                .unwrap();
            f.write_all(&[0xAB, 0xCD, 0xEF]).unwrap();
            f.sync_all().unwrap();
        }

        let vol = EcVolume::new(dir, dir, "", VolumeId(45)).unwrap();
        assert_eq!(vol.read_deleted_needles().unwrap(), ids);
        assert_eq!(
            std::fs::metadata(&ecj_path).unwrap().len(),
            ids.len() as u64 * NEEDLE_ID_SIZE as u64,
            "torn tail should be truncated and journal compacted",
        );
    }

    /// A journal reached through a shared index directory is compacted when
    /// this volume is its only holder: the guard is who holds the path, not
    /// which directory it lives in.
    #[test]
    fn test_shared_index_dir_journal_with_one_holder_is_compacted() {
        let tmp = TempDir::new().unwrap();
        let data_dir = tmp.path().join("data");
        let idx_dir = tmp.path().join("idx");
        std::fs::create_dir_all(&data_dir).unwrap();
        std::fs::create_dir_all(&idx_dir).unwrap();
        let (data, idx) = (data_dir.to_str().unwrap(), idx_dir.to_str().unwrap());

        write_ecx_file(idx, "", VolumeId(44), &[]);
        let ids: Vec<NeedleId> = (1000..1100).map(NeedleId).collect();
        write_bloated_ecj(idx, "", VolumeId(44), &ids, 4096);
        let ecj_path = format!(
            "{}.ecj",
            crate::storage::volume::volume_file_name(idx, "", VolumeId(44))
        );

        let vol = EcVolume::new(data, idx, "", VolumeId(44)).unwrap();
        assert_eq!(vol.file_and_delete_count().1, ids.len() as u64);
        assert_eq!(
            std::fs::metadata(&ecj_path).unwrap().len(),
            (ids.len() * NEEDLE_ID_SIZE) as u64,
        );
        drop(vol);

        let vol2 = EcVolume::new(data, idx, "", VolumeId(44)).unwrap();
        assert_eq!(vol2.read_deleted_needles().unwrap(), ids);
    }

    /// A shard mount can give disk B a volume whose `.ecj` is disk A's. When
    /// A's own volume mounts later, it must not replace the file under B:
    /// B's deletes would go to an unlinked inode and vanish at the next mount.
    #[test]
    fn test_sibling_holder_blocks_compaction() {
        let tmp = TempDir::new().unwrap();
        let a_dir = tmp.path().join("a");
        let b_dir = tmp.path().join("b");
        std::fs::create_dir_all(&a_dir).unwrap();
        std::fs::create_dir_all(&b_dir).unwrap();
        let (a, b) = (a_dir.to_str().unwrap(), b_dir.to_str().unwrap());
        let vid = VolumeId(48);

        write_ecx_file(
            a,
            "",
            vid,
            &[(NeedleId(7), Offset::from_actual_offset(8), Size(10))],
        );
        let ids: Vec<NeedleId> = (1000..1100).map(NeedleId).collect();
        write_bloated_ecj(a, "", vid, &ids, 1);
        let ecj_path = format!(
            "{}.ecj",
            crate::storage::volume::volume_file_name(a, "", vid)
        );

        // B has no index of its own, so it resolves A's and holds A's journal.
        let mut on_b = EcVolume::new(b, a, "", vid).unwrap();
        assert_eq!(on_b.ecj_file_name(), ecj_path);

        // The journal bloats (say, a copy appended to it) and A mounts.
        write_bloated_ecj(a, "", vid, &ids, 4096);
        let bloated = std::fs::metadata(&ecj_path).unwrap().len();
        let on_a = EcVolume::new(a, a, "", vid).unwrap();
        assert_eq!(
            std::fs::metadata(&ecj_path).unwrap().len(),
            bloated,
            "compaction must not replace a journal another volume holds open",
        );

        // B's delete must land in the file at the journal's path.
        on_b.journal_delete(NeedleId(7)).unwrap();
        assert_eq!(std::fs::metadata(&ecj_path).unwrap().len(), bloated + 8);
        drop(on_b);
        drop(on_a);

        // Sole holder now: the next mount compacts and keeps B's delete.
        let vol = EcVolume::new(a, a, "", vid).unwrap();
        let deleted = vol.read_deleted_needles().unwrap();
        assert_eq!(deleted.len(), ids.len() + 1);
        assert!(deleted.contains(&NeedleId(7)));
    }

    /// A copy appending to the journal by path must not have its bytes
    /// dropped by a compaction renaming over the file.
    #[test]
    fn test_active_copy_blocks_compaction() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let vid = VolumeId(49);
        write_ecx_file(dir, "", vid, &[]);
        let ids: Vec<NeedleId> = (1000..1100).map(NeedleId).collect();
        write_bloated_ecj(dir, "", vid, &ids, 4096);
        let ecj_path = format!(
            "{}.ecj",
            crate::storage::volume::volume_file_name(dir, "", vid)
        );
        let bloated = std::fs::metadata(&ecj_path).unwrap().len();

        let copy = crate::storage::erasure_coding::ecj_registry::begin_ecj_write(&ecj_path);
        let vol = EcVolume::new(dir, dir, "", vid).unwrap();
        assert_eq!(std::fs::metadata(&ecj_path).unwrap().len(), bloated);
        drop(vol);
        drop(copy);

        let vol = EcVolume::new(dir, dir, "", vid).unwrap();
        assert_eq!(
            std::fs::metadata(&ecj_path).unwrap().len(),
            (ids.len() * NEEDLE_ID_SIZE) as u64,
        );
        assert_eq!(vol.read_deleted_needles().unwrap(), ids);
    }

    /// A journal that grew, or was replaced, after it was loaded must not be
    /// compacted from the stale set.
    #[test]
    fn test_journal_changed_after_load_is_detected() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let vid = VolumeId(50);
        write_ecx_file(dir, "", vid, &[]);
        write_bloated_ecj(dir, "", vid, &[NeedleId(1)], 1);

        let vol = EcVolume::new(dir, dir, "", vid).unwrap();
        let ecj_path = vol.ecj_file_name();
        assert!(vol.ecj_unchanged_since_load(&ecj_path).unwrap());

        {
            let mut f = OpenOptions::new().append(true).open(&ecj_path).unwrap();
            f.write_all(&[0u8; NEEDLE_ID_SIZE]).unwrap();
        }
        assert!(!vol.ecj_unchanged_since_load(&ecj_path).unwrap());

        // Same size, different inode: a copy that replaced the file.
        let replacement = format!("{}.replacement", ecj_path);
        std::fs::write(&replacement, [0u8; NEEDLE_ID_SIZE]).unwrap();
        std::fs::rename(&replacement, &ecj_path).unwrap();
        #[cfg(unix)]
        assert!(!vol.ecj_unchanged_since_load(&ecj_path).unwrap());
    }

    /// A compaction tmp left by a crash between its write and the rename is
    /// removed at the next mount, even when the journal needs no compaction.
    #[test]
    fn test_stale_compaction_tmp_is_removed_at_mount() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let vid = VolumeId(51);
        write_ecx_file(dir, "", vid, &[]);
        write_bloated_ecj(dir, "", vid, &[NeedleId(1)], 1);
        let tmp_path = format!(
            "{}{}",
            crate::storage::volume::volume_file_name(dir, "", vid),
            ECJ_COMPACT_TMP_EXT
        );
        std::fs::write(&tmp_path, [0u8; 64]).unwrap();

        let mut vol = EcVolume::new(dir, dir, "", vid).unwrap();
        assert!(!std::path::Path::new(&tmp_path).exists());

        // destroy() sweeps it too.
        std::fs::write(&tmp_path, [0u8; 64]).unwrap();
        vol.destroy();
        assert!(!std::path::Path::new(&tmp_path).exists());
    }

    /// A mount the bitrot sidecar refuses must leave the journal as it was:
    /// compaction runs only after every check that can fail the mount.
    #[test]
    fn test_refused_mount_does_not_compact() {
        use crate::pb::volume_server_pb::{ChecksumAlgorithm, EcBitrotProtection};
        use crate::storage::erasure_coding::ec_bitrot;
        use crate::storage::volume::{VifEcShardConfig, VifVolumeInfo};

        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let vid = VolumeId(52);
        let base = crate::storage::volume::volume_file_name(dir, "", vid);
        write_ecx_file(dir, "", vid, &[]);
        let ids: Vec<NeedleId> = (1000..1100).map(NeedleId).collect();
        write_bloated_ecj(dir, "", vid, &ids, 4096);
        std::fs::write(
            format!("{}.vif", base),
            serde_json::to_string(&VifVolumeInfo {
                version: 3,
                ec_shard_config: Some(VifEcShardConfig {
                    data_shards: 10,
                    parity_shards: 4,
                    ..Default::default()
                }),
                ..Default::default()
            })
            .unwrap(),
        )
        .unwrap();
        // A generation-0 sidecar recording a different layout: mount refused.
        ec_bitrot::save_bitrot_sidecar(
            &ec_bitrot::bitrot_sidecar_path(&base, 0),
            &EcBitrotProtection {
                algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32,
                block_size: ec_bitrot::DEFAULT_BITROT_BLOCK_SIZE as u32,
                generation: 0,
                ec_shard_config: Some(ec_bitrot::ec_shard_config(12, 4, 0)),
                shards: Vec::new(),
                encode_uuid: vec![0u8; 16],
            },
        )
        .unwrap();
        let ecj_path = format!("{}.ecj", base);
        let before = std::fs::read(&ecj_path).unwrap();

        let err = EcVolume::new(dir, dir, "", vid)
            .err()
            .expect("a geometry-mismatched sidecar must refuse the mount");
        assert!(err.to_string().contains("refusing to serve"), "{}", err);
        assert_eq!(std::fs::read(&ecj_path).unwrap(), before);
    }

    /// Compaction must never leave a handle that does not refer to the
    /// journal's pathname, and must not leave its temp file behind.
    #[test]
    fn test_compaction_leaves_a_usable_handle_and_no_temp_file() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        write_ecx_file(
            dir,
            "",
            VolumeId(43),
            &[(NeedleId(7), Offset::from_actual_offset(8), Size(10))],
        );
        let ids: Vec<NeedleId> = (1000..1100).map(NeedleId).collect();
        write_bloated_ecj(dir, "", VolumeId(43), &ids, 4096);

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(43)).unwrap();
        assert!(vol.ecj_file.is_some(), "mount must leave a journal handle");

        let ecj_path = format!(
            "{}.ecj",
            crate::storage::volume::volume_file_name(dir, "", VolumeId(43))
        );
        assert!(!std::path::Path::new(&format!("{}.compact.tmp", ecj_path)).exists());

        vol.journal_delete(NeedleId(7)).unwrap();
        let on_disk = std::fs::metadata(&ecj_path).unwrap().len();
        assert_eq!(
            on_disk,
            ((ids.len() + 1) * NEEDLE_ID_SIZE) as u64,
            "post-compaction append did not reach the journal's pathname",
        );
    }

    /// Mount a volume `vid` over a bloated journal with the given compaction
    /// filesystem steps. Returns the mount result, the journal path, and the
    /// journal's bytes before the mount.
    fn mount_bloated_with(
        dir: &str,
        vid: VolumeId,
        ops: EcjFsOps,
    ) -> (io::Result<EcVolume>, String, Vec<u8>) {
        write_ecx_file(
            dir,
            "",
            vid,
            &[(NeedleId(7), Offset::from_actual_offset(8), Size(10))],
        );
        let ids: Vec<NeedleId> = (1000..1100).map(NeedleId).collect();
        write_bloated_ecj(dir, "", vid, &ids, 4096);
        let ecj_path = format!(
            "{}.ecj",
            crate::storage::volume::volume_file_name(dir, "", vid)
        );
        let before = std::fs::read(&ecj_path).unwrap();
        (
            EcVolume::open_with(dir, dir, "", vid, ops),
            ecj_path,
            before,
        )
    }

    fn failing_rename(_: &str, _: &str) -> io::Result<()> {
        Err(io::Error::new(
            io::ErrorKind::PermissionDenied,
            "injected rename failure",
        ))
    }

    fn failing_reopen(_: &str) -> io::Result<File> {
        Err(io::Error::new(
            io::ErrorKind::NotFound,
            "injected reopen failure",
        ))
    }

    /// A failed rename publishes nothing: the mount succeeds on the original
    /// journal, byte for byte, with no tmp left, and deletes still reach the
    /// file at the journal's path.
    #[test]
    fn test_compaction_rename_failure_keeps_original_journal() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let ops = EcjFsOps {
            rename: failing_rename,
            ..EcjFsOps::REAL
        };
        let (vol, ecj_path, before) = mount_bloated_with(dir, VolumeId(46), ops);
        let mut vol = vol.expect("a failed rename must not fail the mount");

        assert_eq!(std::fs::read(&ecj_path).unwrap(), before);
        assert!(!std::path::Path::new(&format!("{}.compact.tmp", ecj_path)).exists());
        assert_eq!(vol.file_and_delete_count().1, 100);

        vol.journal_delete(NeedleId(7)).unwrap();
        assert_eq!(
            std::fs::metadata(&ecj_path).unwrap().len(),
            before.len() as u64 + NEEDLE_ID_SIZE as u64,
            "the restored handle must append to the original journal",
        );
    }

    /// A failed rename whose handle restore also fails leaves no usable
    /// handle: the mount fails, reports both errors, and says nothing was
    /// compacted. The original journal is untouched.
    #[test]
    fn test_compaction_rename_and_restore_failure_is_mount_error() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let ops = EcjFsOps {
            rename: failing_rename,
            reopen: failing_reopen,
            ..EcjFsOps::REAL
        };
        let (vol, ecj_path, before) = mount_bloated_with(dir, VolumeId(53), ops);
        let msg = vol
            .err()
            .expect("mount must fail without a handle")
            .to_string();

        assert!(msg.contains("no usable journal handle"), "{}", msg);
        assert!(msg.contains("injected rename failure"), "{}", msg);
        assert!(msg.contains("injected reopen failure"), "{}", msg);
        assert!(msg.contains("reopening the original journal"), "{}", msg);
        assert_eq!(std::fs::read(&ecj_path).unwrap(), before);
        assert!(!std::path::Path::new(&format!("{}.compact.tmp", ecj_path)).exists());
    }

    /// A reopen failure after the rename fails the mount: the journal was
    /// replaced and the volume has no handle to it.
    #[test]
    fn test_compaction_post_rename_reopen_failure_is_mount_error() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let ops = EcjFsOps {
            reopen: failing_reopen,
            ..EcjFsOps::REAL
        };
        let (vol, ecj_path, _) = mount_bloated_with(dir, VolumeId(47), ops);
        let msg = vol
            .err()
            .expect("mount must fail without a handle")
            .to_string();

        assert!(msg.contains("no usable journal handle"), "{}", msg);
        assert!(msg.contains("could not reopen it"), "{}", msg);
        assert!(msg.contains("injected reopen failure"), "{}", msg);
        assert_eq!(
            std::fs::metadata(&ecj_path).unwrap().len(),
            100 * NEEDLE_ID_SIZE as u64,
            "the rename had published the compacted journal",
        );
    }

    /// A directory fsync failure after the rename fails the mount too: the
    /// replacement may not survive a power loss.
    #[test]
    fn test_compaction_post_rename_fsync_failure_is_mount_error() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let ops = EcjFsOps {
            fsync_dir: |_| Err(io::Error::other("injected fsync failure")),
            ..EcjFsOps::REAL
        };
        let (vol, _, _) = mount_bloated_with(dir, VolumeId(54), ops);
        let msg = vol.err().expect("mount must fail").to_string();
        assert!(msg.contains("could not fsync its directory"), "{}", msg);
        assert!(msg.contains("injected fsync failure"), "{}", msg);
    }

    /// The directory sync after the rename must report a directory it cannot
    /// open. The crate's best-effort `fsync_dir` returns Ok there, which would
    /// let the mount take deletes against an entry that was never synced.
    #[test]
    fn test_fsync_ecj_dir_fails_when_directory_cannot_be_opened() {
        let tmp = TempDir::new().unwrap();
        let missing = tmp.path().join("gone").join("1.ecj");
        let missing = missing.to_str().unwrap();
        #[cfg(not(windows))]
        assert!(fsync_ecj_dir(missing).is_err());
        assert!(crate::storage::volume::fsync_dir(missing).is_ok());
    }

    /// The real mount over a directory it may write and search but not read:
    /// the rename succeeds and the directory sync cannot open the directory,
    /// so the mount fails rather than serving an unsynced replacement.
    #[cfg(unix)]
    #[test]
    fn test_compaction_unreadable_directory_is_mount_error() {
        use std::os::unix::fs::PermissionsExt;
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let set_mode =
            |mode| std::fs::set_permissions(dir, std::fs::Permissions::from_mode(mode)).unwrap();
        write_ecx_file(
            dir,
            "",
            VolumeId(55),
            &[(NeedleId(7), Offset::from_actual_offset(8), Size(10))],
        );
        let ids: Vec<NeedleId> = (1000..1100).map(NeedleId).collect();
        write_bloated_ecj(dir, "", VolumeId(55), &ids, 4096);

        set_mode(0o300);
        if File::open(dir).is_ok() {
            // Root opens it regardless; nothing to test.
            set_mode(0o755);
            return;
        }
        let result = EcVolume::new(dir, dir, "", VolumeId(55));
        set_mode(0o755);
        let msg = result
            .err()
            .expect("an unsynced replacement must fail the mount")
            .to_string();
        assert!(msg.contains("could not fsync its directory"), "{}", msg);
    }

    thread_local! {
        /// The journal path and replacement bytes `rewrite_after_last_read`
        /// writes; taken when it does.
        static IN_PLACE_REWRITE: std::cell::RefCell<Option<(String, Vec<u8>)>> =
            const { std::cell::RefCell::new(None) };
    }

    /// A load step that, once the last chunk is read, truncates and refills
    /// the journal in place as a registered writer, start to finish, the way a
    /// `ReceiveFile` would.
    fn rewrite_after_last_read(f: &File, buf: &mut [u8], off: u64) -> io::Result<()> {
        read_exact_at(f, buf, off)?;
        let pending = IN_PLACE_REWRITE.with(|r| {
            let mut r = r.borrow_mut();
            match r.as_ref() {
                Some((_, bytes)) if off + buf.len() as u64 >= bytes.len() as u64 => r.take(),
                _ => None,
            }
        });
        if let Some((path, bytes)) = pending {
            let _write = crate::storage::erasure_coding::ecj_registry::begin_ecj_write(&path);
            let mut out = OpenOptions::new().write(true).truncate(true).open(&path)?;
            out.write_all(&bytes)?;
        }
        Ok(())
    }

    /// A `ReceiveFile` that runs during the mount's load and finishes before
    /// compaction can leave the same inode at the same length with different
    /// ids, which the inode-and-size re-check accepts. Compacting would then
    /// overwrite the received ids with the stale set.
    #[test]
    fn test_in_place_rewrite_during_load_blocks_compaction() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let vid = VolumeId(56);
        write_ecx_file(
            dir,
            "",
            vid,
            &[(NeedleId(7), Offset::from_actual_offset(8), Size(10))],
        );
        let ids: Vec<NeedleId> = (1000..1100).map(NeedleId).collect();
        write_bloated_ecj(dir, "", vid, &ids, 4096);
        let ecj_path = format!(
            "{}.ecj",
            crate::storage::volume::volume_file_name(dir, "", vid)
        );
        let mut one = vec![0u8; 100 * NEEDLE_ID_SIZE];
        for (i, entry) in one.chunks_exact_mut(NEEDLE_ID_SIZE).enumerate() {
            NeedleId(2000 + i as u64).to_bytes(entry);
        }
        let rewrite = one.repeat(4096);
        IN_PLACE_REWRITE.with(|r| *r.borrow_mut() = Some((ecj_path.clone(), rewrite.clone())));

        let ops = EcjFsOps {
            read_at: rewrite_after_last_read,
            ..EcjFsOps::REAL
        };
        let vol = EcVolume::open_with(dir, dir, "", vid, ops).unwrap();
        assert!(
            IN_PLACE_REWRITE.with(|r| r.borrow().is_none()),
            "the rewrite never ran"
        );
        assert!(
            std::fs::read(&ecj_path).unwrap() == rewrite,
            "the received journal must not be compacted from the set loaded before it",
        );
        drop(vol);
    }

    #[test]
    fn test_journal_delete_wrong_cookie() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let needle = NeedleId(7);
        let entries = vec![(needle, Offset::from_actual_offset(8), Size(100))];
        write_ecx_file(dir, "", VolumeId(1), &entries);

        let vif = crate::storage::volume::VifVolumeInfo {
            dat_file_size: 14000,
            ..Default::default()
        };
        let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(1));
        std::fs::write(
            format!("{}.vif", base),
            serde_json::to_string_pretty(&vif).unwrap(),
        )
        .unwrap();

        let mut shard9 = EcVolumeShard::new(dir, "", VolumeId(1), 9);
        shard9.create().unwrap();
        shard9.write_all(&[0xAAu8; 2048]).unwrap();
        shard9.close();

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        vol.add_shard(EcVolumeShard::new(dir, "", VolumeId(1), 9))
            .unwrap();

        let (off, size) = vol
            .find_needle_from_ecx(needle)
            .unwrap()
            .expect("fixture needle must be indexed");
        let intervals = vol.locate_ec_shard_needle_interval(off.to_actual_offset(), size);
        assert!(
            !intervals.is_empty(),
            "fixture must locate to a shard for the test to be meaningful"
        );
        let (located, _) = vol.interval_to_shard_id_and_offset(&intervals[0]);
        assert_ne!(
            located, 9,
            "fixture must locate away from mounted shard 9, got {}",
            located
        );

        let res = vol.journal_delete_with_cookie(needle, Cookie(0xDEAD_BEEF));
        let err = res.expect_err("wrong cookie must Err, not bypass to journal");
        assert!(
            err.to_string().contains("cannot verify cookie"),
            "fail-closed error must say why, got: {}",
            err
        );
        let deleted = vol.read_deleted_needles().unwrap();
        assert!(
            !deleted.contains(&needle),
            "failed delete must not append to .ecj, got {:?}",
            deleted
        );

        vol.journal_delete_with_cookie(needle, Cookie(0)).unwrap();
        let deleted = vol.read_deleted_needles().unwrap();
        assert!(
            deleted.contains(&needle),
            "cookie-0 delete must still journal, got {:?}",
            deleted
        );
    }

    #[test]
    fn test_journal_delete_incomplete_header() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let needle = NeedleId(7);
        let entries = vec![(needle, Offset::from_actual_offset(8), Size(100))];
        write_ecx_file(dir, "", VolumeId(1), &entries);

        let vif = crate::storage::volume::VifVolumeInfo {
            dat_file_size: 14000,
            ..Default::default()
        };
        let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(1));
        std::fs::write(
            format!("{}.vif", base),
            serde_json::to_string_pretty(&vif).unwrap(),
        )
        .unwrap();

        let probe = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        let (off, size) = probe
            .find_needle_from_ecx(needle)
            .unwrap()
            .expect("fixture needle must be indexed");
        let intervals = probe.locate_ec_shard_needle_interval(off.to_actual_offset(), size);
        assert!(
            !intervals.is_empty(),
            "fixture must locate to a shard for the test to be meaningful"
        );
        let (located, located_offset) = probe.interval_to_shard_id_and_offset(&intervals[0]);
        drop(probe);

        let mut shard = EcVolumeShard::new(dir, "", VolumeId(1), located);
        shard.create().unwrap();
        shard.write_all(&[0x00u8; 2]).unwrap();
        shard.close();

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        vol.add_shard(EcVolumeShard::new(dir, "", VolumeId(1), located))
            .unwrap();

        let shard_ref = vol.shards[located as usize]
            .as_ref()
            .expect("located shard must be mounted");
        assert!(
            (shard_ref.file_size()) < located_offset + 4,
            "fixture must truncate the header read (file {} bytes, offset {})",
            shard_ref.file_size(),
            located_offset
        );

        let res = vol.journal_delete_with_cookie(needle, Cookie(0x1234));
        let err = res.expect_err("short header read must Err, not forge-match");
        assert!(
            err.to_string().contains("incomplete header"),
            "short read must report incomplete header, got: {}",
            err
        );
        let deleted = vol.read_deleted_needles().unwrap();
        assert!(
            !deleted.contains(&needle),
            "failed delete must not append to .ecj, got {:?}",
            deleted
        );

        vol.journal_delete_with_cookie(needle, Cookie(0)).unwrap();
        let deleted = vol.read_deleted_needles().unwrap();
        assert!(
            deleted.contains(&needle),
            "cookie-0 delete must still journal, got {:?}",
            deleted
        );
    }

    #[test]
    fn test_ec_volume_shard_bits() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        write_ecx_file(dir, "", VolumeId(1), &[]);

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        assert_eq!(vol.shard_count(), 0);

        // Create a shard file so we can add it
        let mut shard = EcVolumeShard::new(dir, "", VolumeId(1), 3);
        shard.create().unwrap();
        shard.write_all(&[0u8; 100]).unwrap();
        shard.close();

        vol.add_shard(EcVolumeShard::new(dir, "", VolumeId(1), 3))
            .unwrap();
        assert_eq!(vol.shard_count(), 1);
        assert!(vol.shard_bits().has_shard_id(3));
    }

    /// A 0-byte shard file beside an index WITH entries is residue of a
    /// failed copy: `add_shard` must refuse it and leave the slot
    /// unregistered, so no size-0 claim is advertised (and no re-copy is
    /// attracted to a file that was never valid). Beside a 0-byte index it
    /// is the legitimate empty-volume layout and must keep mounting. The
    /// startup scan already skips such files; this covers the mount path.
    /// Mirrors TestLoadEcShardRefusesEmptyShardFile in Go.
    #[test]
    fn test_add_shard_refuses_empty_shard_file() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        write_ecx_file(
            dir,
            "",
            VolumeId(1),
            &[(NeedleId(1), Offset::from_actual_offset(8), Size(100))],
        );

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();

        let mut shard = EcVolumeShard::new(dir, "", VolumeId(1), 4);
        shard.create().unwrap();
        shard.close();

        let err = vol
            .add_shard(EcVolumeShard::new(dir, "", VolumeId(1), 4))
            .expect_err("adding a 0-byte shard file must fail when the index has entries");
        assert!(
            err.to_string().contains("empty (0 bytes)"),
            "a 0-byte shard should be refused as empty, got: {}",
            err
        );
        assert_eq!(vol.shard_count(), 0, "a refused shard must not register");
        assert!(!vol.shard_bits().has_shard_id(4));
    }

    /// The legitimate empty-volume layout: a volume encoded with no live
    /// needles has a 0-byte .ecx AND 0-byte shards, and mounting it must
    /// keep working.
    #[test]
    fn test_add_shard_accepts_empty_shard_of_empty_volume() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        write_ecx_file(dir, "", VolumeId(1), &[]);

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();

        let mut shard = EcVolumeShard::new(dir, "", VolumeId(1), 4);
        shard.create().unwrap();
        shard.close();

        vol.add_shard(EcVolumeShard::new(dir, "", VolumeId(1), 4))
            .expect("a 0-byte shard of an empty volume (0-byte index) must mount");
        assert!(vol.shard_bits().has_shard_id(4));
    }

    #[test]
    fn test_ec_volume_uses_collection_prefixed_vif_config() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        write_ecx_file(dir, "pics", VolumeId(1), &[]);

        let vif = crate::storage::volume::VifVolumeInfo {
            ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
                data_shards: 6,
                parity_shards: 3,
                ..Default::default()
            }),
            ..Default::default()
        };
        let base = crate::storage::volume::volume_file_name(dir, "pics", VolumeId(1));
        std::fs::write(
            format!("{}.vif", base),
            serde_json::to_string_pretty(&vif).unwrap(),
        )
        .unwrap();

        let vol = EcVolume::new(dir, dir, "pics", VolumeId(1)).unwrap();
        assert_eq!(vol.data_shards, 6);
        assert_eq!(vol.parity_shards, 3);
    }

    /// A vif that RECORDS a ratio no volume could have must fail the mount.
    /// Substituting the default 10+4 (and with it the legacy block layout)
    /// would read a uniform volume's shards at the wrong offsets and answer
    /// with the wrong bytes; only an entirely absent config means "this
    /// predates the record", which
    /// `test_ec_volume_absent_vif_config_uses_defaults` covers.
    #[test]
    fn test_ec_volume_invalid_vif_config_fails_the_mount() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        write_ecx_file(dir, "pics", VolumeId(1), &[]);

        // data + parity exceeds MAX_SHARD_COUNT, so the config is rejected.
        let vif = crate::storage::volume::VifVolumeInfo {
            ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
                data_shards: 20,
                parity_shards: 20,
                ..Default::default()
            }),
            ..Default::default()
        };
        let base = crate::storage::volume::volume_file_name(dir, "pics", VolumeId(1));
        std::fs::write(
            format!("{}.vif", base),
            serde_json::to_string_pretty(&vif).unwrap(),
        )
        .unwrap();

        // EcVolume has no Debug impl, so match rather than expect_err.
        match EcVolume::new(dir, dir, "pics", VolumeId(1)) {
            Ok(_) => panic!("an impossible ratio must fail the mount"),
            Err(e) => assert_eq!(e.kind(), io::ErrorKind::InvalidData, "got {e}"),
        }
    }

    /// The compatibility case the check above must not swallow: a vif with no
    /// EC config at all predates the record, and mounts on the defaults.
    #[test]
    fn test_ec_volume_absent_vif_config_uses_defaults() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        write_ecx_file(dir, "pics", VolumeId(1), &[]);
        let base = crate::storage::volume::volume_file_name(dir, "pics", VolumeId(1));
        std::fs::write(format!("{}.vif", base), r#"{"version":3}"#).unwrap();

        let vol = EcVolume::new(dir, dir, "pics", VolumeId(1)).unwrap();
        assert_eq!(vol.data_shards, DATA_SHARDS_COUNT as u32);
        assert_eq!(vol.parity_shards, PARITY_SHARDS_COUNT as u32);
        assert_eq!(vol.block_size, 0, "legacy layout for a pre-record volume");
    }

    #[test]
    fn test_ec_volume_wide_ratio_vif_config() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        write_ecx_file(dir, "pics", VolumeId(1), &[]);

        // A wider-than-default ratio within MAX_SHARD_COUNT must load as-is.
        let vif = crate::storage::volume::VifVolumeInfo {
            ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
                data_shards: 16,
                parity_shards: 4,
                ..Default::default()
            }),
            ..Default::default()
        };
        let base = crate::storage::volume::volume_file_name(dir, "pics", VolumeId(1));
        std::fs::write(
            format!("{}.vif", base),
            serde_json::to_string_pretty(&vif).unwrap(),
        )
        .unwrap();

        let vol = EcVolume::new(dir, dir, "pics", VolumeId(1)).unwrap();
        assert_eq!(vol.data_shards, 16);
        assert_eq!(vol.parity_shards, 4);
    }

    #[test]
    fn test_scrub_local_skips_tombstones() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let entries = vec![(NeedleId(1), Offset::from_actual_offset(0), Size(-1))];
        write_ecx_file(dir, "", VolumeId(1), &entries);

        let vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        let (count, broken, errs) = vol.scrub_local();
        assert_eq!(count, 1);
        assert!(broken.is_empty(), "{:?}", broken);
        assert!(errs.is_empty(), "{:?}", errs);
    }

    #[test]
    fn test_scrub_local_clean_when_no_local_shards() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let entries = vec![(NeedleId(1), Offset::from_actual_offset(0), Size(100))];
        write_ecx_file(dir, "", VolumeId(1), &entries);

        // dat_file_size makes the shard-size math well-defined.
        let vif = crate::storage::volume::VifVolumeInfo {
            dat_file_size: 14000,
            ..Default::default()
        };
        let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(1));
        std::fs::write(
            format!("{}.vif", base),
            serde_json::to_string_pretty(&vif).unwrap(),
        )
        .unwrap();

        // No shard files present: nothing to verify locally, so no errors.
        let vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        let (count, broken, errs) = vol.scrub_local();
        assert_eq!(count, 1);
        assert!(broken.is_empty(), "{:?}", broken);
        assert!(errs.is_empty(), "{:?}", errs);
    }

    #[test]
    fn test_scrub_local_suppresses_delete_state_disagreement() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        // Live index entry (size > 0) whose reassembled on-disk header reports size 0
        // (deleted-on-shards but live-in-index) — a delete-state disagreement, not corruption.
        let entries = vec![(NeedleId(1), Offset::from_actual_offset(0), Size(100))];
        write_ecx_file(dir, "", VolumeId(1), &entries);

        let vif = crate::storage::volume::VifVolumeInfo {
            dat_file_size: 14000,
            ..Default::default()
        };
        let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(1));
        std::fs::write(
            format!("{}.vif", base),
            serde_json::to_string_pretty(&vif).unwrap(),
        )
        .unwrap();

        // Shard 0 holds the needle's bytes; all-zero so the parsed header size is 0.
        let mut shard0 = EcVolumeShard::new(dir, "", VolumeId(1), 0);
        shard0.create().unwrap();
        shard0.write_all(&[0u8; 256]).unwrap();
        shard0.close();

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        vol.add_shard(EcVolumeShard::new(dir, "", VolumeId(1), 0))
            .unwrap();

        let (count, broken, errs) = vol.scrub_local();
        assert_eq!(count, 1);
        assert!(broken.is_empty(), "{:?}", broken);
        assert!(
            errs.is_empty(),
            "delete-state disagreement must be suppressed, got {:?}",
            errs
        );
    }

    #[test]
    fn test_scrub_local_reports_genuine_size_corruption() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let entries = vec![(NeedleId(1), Offset::from_actual_offset(0), Size(100))];
        write_ecx_file(dir, "", VolumeId(1), &entries);

        let vif = crate::storage::volume::VifVolumeInfo {
            dat_file_size: 14000,
            ..Default::default()
        };
        let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(1));
        std::fs::write(
            format!("{}.vif", base),
            serde_json::to_string_pretty(&vif).unwrap(),
        )
        .unwrap();

        // On-disk header reports a non-zero size (50) that disagrees with the live
        // index size (100): genuine corruption, not a delete-state race — must report.
        let mut bytes = vec![0u8; 256];
        bytes[15] = 50; // header size field (big-endian u32) = 50
        let mut shard0 = EcVolumeShard::new(dir, "", VolumeId(1), 0);
        shard0.create().unwrap();
        shard0.write_all(&bytes).unwrap();
        shard0.close();

        let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        vol.add_shard(EcVolumeShard::new(dir, "", VolumeId(1), 0))
            .unwrap();

        let (_count, _broken, errs) = vol.scrub_local();
        assert!(
            !errs.is_empty(),
            "a non-zero size mismatch is genuine corruption and must be reported"
        );
    }

    /// Build one real encoded EC volume in `dir`, then hand back N runtimes over
    /// it, each holding the shard ids it was given. Models the reconciled
    /// split-disk mount without needing N directories.
    fn split_runtimes(dir: &str, vid: VolumeId, subsets: &[&[u8]]) -> Vec<EcVolume> {
        use crate::storage::needle_map::NeedleMapKind;
        use crate::storage::volume::{Volume, VolumeSpec};

        let mut v = Volume::new(
            dir,
            dir,
            vid,
            NeedleMapKind::InMemory,
            &VolumeSpec::default(),
        )
        .unwrap();
        for i in 1..=8 {
            let data = format!("test data for needle {} with a bit more length", i);
            let mut n = Needle {
                id: NeedleId(i),
                cookie: Cookie(i as u32),
                data: data.as_bytes().to_vec(),
                data_size: data.len() as u32,
                ..Needle::default()
            };
            v.write_needle(&mut n, true, false).unwrap();
        }
        v.sync_to_disk().unwrap();
        v.close();
        crate::storage::erasure_coding::ec_encoder::write_ec_files(dir, dir, "", vid, 10, 4)
            .unwrap();

        subsets
            .iter()
            .map(|ids| {
                let mut ev = EcVolume::new(dir, dir, "", vid).unwrap();
                for &id in ids.iter() {
                    ev.add_shard(EcVolumeShard::new(dir, "", vid, id)).unwrap();
                }
                ev
            })
            .collect()
    }

    /// The union must reach every disk's shards, and first-disk-wins must
    /// resolve a shard mounted on two disks — the same rule
    /// `collect_ec_shard_dirs` and Go's `CollectEcShards` already use.
    #[test]
    fn test_merge_ec_runtimes_unions_slots_first_disk_wins() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        // Disk 0: 0..=6 plus 9 (9 is also on disk 1 — a duplicate mount).
        // Disk 1: 7..=13.
        let runtimes = split_runtimes(
            dir,
            VolumeId(1),
            &[&[0, 1, 2, 3, 4, 5, 6, 9], &[7, 8, 9, 10, 11, 12, 13]],
        );
        let refs: Vec<&EcVolume> = runtimes.iter().collect();

        let merged = merge_ec_runtimes(&refs).expect("non-empty input merges");
        assert_eq!(merged.merged.len(), 2, "both runtimes share generation 0");
        assert!(
            merged.skipped.is_empty(),
            "nothing to skip: {:?}",
            merged.skipped
        );

        // Every shard 0..=13 is reachable from the merged slots.
        for id in 0..14usize {
            assert!(
                merged.slots[id].is_some(),
                "shard {} missing from the union",
                id
            );
        }
        // Shard 9 is held by both; the first disk wins.
        let (owner, _) = merged.slots[9].unwrap();
        assert!(
            std::ptr::eq(owner, refs[0]),
            "duplicate shard must resolve to the first disk"
        );
        // Slots are shard-id indexed across the volume's whole shard space,
        // never compacted: index N is shard N or nothing.
        assert_eq!(
            merged.slots.len(),
            14,
            "slots must span the full 10+4 shard space"
        );
    }

    /// Leniency is keyed on the ANCHOR, never the holder: a known identity must
    /// not accept an unstamped holder (store_ec.rs:667-673,
    /// grpc_server.rs:3743-3748). Merging leftover legacy shards beside a
    /// current encode is what produces false corruption reports.
    #[test]
    fn test_merge_ec_runtimes_excludes_incompatible_identities() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let mut runtimes = split_runtimes(
            dir,
            VolumeId(1),
            &[&[0, 1, 2, 3, 4, 5, 6], &[7, 8, 9], &[10, 11, 12, 13]],
        );
        runtimes[0].encode_ts_ns = 500; // stale, but stamped
        runtimes[1].encode_ts_ns = 0; // legacy, unstamped
        runtimes[2].encode_ts_ns = 900; // the live encode run
        let refs: Vec<&EcVolume> = runtimes.iter().collect();

        let merged = merge_ec_runtimes(&refs).unwrap();

        // Anchor is the newest known identity; only exact matches merge.
        assert_eq!(merged.anchor.encode_ts_ns, 900);
        assert_eq!(merged.merged.len(), 1);
        assert!(
            merged.slots[10].is_some(),
            "the anchor's own shards must merge"
        );
        assert!(
            merged.slots[0].is_none(),
            "an older known generation must not merge"
        );
        assert!(
            merged.slots[7].is_none(),
            "an unstamped holder must not merge"
        );

        // Exclusions are REPORTED, never silent — that is what keeps this from
        // regressing to the silent-clean bug this whole change fixes.
        assert_eq!(merged.skipped.len(), 2, "got {:?}", merged.skipped);
        assert!(merged.skipped.iter().all(|s| s.contains("not verified")));
    }

    /// When no runtime carries an identity there is nothing to fence on, so
    /// behavior stays exactly as it is today: everything merges.
    #[test]
    fn test_merge_ec_runtimes_is_lenient_when_no_identity_is_known() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let runtimes = split_runtimes(dir, VolumeId(1), &[&[0, 1, 2], &[3, 4, 5]]);
        assert!(runtimes.iter().all(|r| r.encode_ts_ns == 0));
        let refs: Vec<&EcVolume> = runtimes.iter().collect();

        let merged = merge_ec_runtimes(&refs).unwrap();
        assert_eq!(merged.merged.len(), 2);
        assert!(merged.skipped.is_empty());
        assert!(merged.slots[3].is_some());
    }

    /// The anchor supplies the volume-level metadata, so prefer a shard-bearing
    /// runtime at the anchor generation over an empty one.
    #[test]
    fn test_merge_ec_runtimes_anchor_prefers_a_shard_bearing_runtime() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let runtimes = split_runtimes(dir, VolumeId(1), &[&[], &[0, 1, 2]]);
        let refs: Vec<&EcVolume> = runtimes.iter().collect();

        let merged = merge_ec_runtimes(&refs).unwrap();
        assert!(
            std::ptr::eq(merged.anchor, refs[1]),
            "anchor must hold shards"
        );
        assert!(
            merged.skipped.is_empty(),
            "same generation: nothing is excluded"
        );
    }

    /// Empty input is the vanished-volume case and the ONLY None.
    #[test]
    fn test_merge_ec_runtimes_none_only_when_empty() {
        assert!(merge_ec_runtimes(&[]).is_none());
    }

    /// The vanished-volume case: callers' `let ... else` guards depend on this
    /// being the ONLY None.
    #[test]
    fn test_for_volumes_is_none_only_for_an_empty_slice() {
        assert!(EcChecksumScrubPlan::for_volumes(&[]).is_none());
        assert!(EcLocalScrubPlan::for_volumes(&[]).is_none());
    }

    /// The whole point: corruption on a shard that only a sibling runtime holds
    /// must be reported. Before this change the scrub saw disk 0 alone and
    /// returned clean.
    #[test]
    fn test_checksum_scrub_for_volumes_catches_sibling_disk_corruption() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        // Shard 11 is held ONLY by the second runtime.
        let runtimes = split_runtimes(
            dir,
            VolumeId(1),
            &[&[0, 1, 2, 3, 4, 5, 6], &[7, 8, 9, 10, 11, 12, 13]],
        );
        let refs: Vec<&EcVolume> = runtimes.iter().collect();

        // Clean to start.
        let (scanned, broken, errs) = EcChecksumScrubPlan::for_volumes(&refs).unwrap().run();
        assert!(errs.is_empty(), "unexpected errors: {:?}", errs);
        assert!(broken.is_empty(), "unexpected mismatches: {:?}", broken);
        assert!(scanned > 0, "merged plan scanned nothing");

        // Corrupt shard 11 — reachable only through the second runtime.
        let shard11 = format!("{}/1.ec11", dir);
        let mut bytes = std::fs::read(&shard11).unwrap();
        bytes[0] ^= 0xFF;
        std::fs::write(&shard11, &bytes).unwrap();

        let (_, broken2, _) = EcChecksumScrubPlan::for_volumes(&refs).unwrap().run();
        assert!(
            broken2.contains(&11),
            "sibling-disk corruption must be flagged, got {:?}",
            broken2
        );

        // The single-runtime path is exactly the old blind spot: disk 0 alone
        // still reports clean, which is why aggregation was needed.
        let (_, broken_disk0, _) = refs[0].checksum_scrub();
        assert!(!broken_disk0.contains(&11));
    }

    /// An excluded runtime is reported through the plan's errors, but ONLY on
    /// the On path — the Off arm's clean-empty contract is Go parity
    /// (`case BitrotOff: return 0, nil, nil`) and must not grow errors.
    #[test]
    fn test_checksum_scrub_skips_are_reported_but_never_break_the_off_arm() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let mut runtimes = split_runtimes(dir, VolumeId(1), &[&[0, 1, 2], &[3, 4, 5]]);
        runtimes[0].encode_ts_ns = 100; // excluded: older known identity
        runtimes[1].encode_ts_ns = 200; // anchor
        let refs: Vec<&EcVolume> = runtimes.iter().collect();

        // With protection ON, the skip surfaces as an error line, and the
        // surviving (anchor) runtime is still actually scanned.
        let (scanned, _, errs) = EcChecksumScrubPlan::for_volumes(&refs).unwrap().run();
        assert!(
            scanned > 0,
            "excluding one runtime must not stop the rest from being scanned"
        );
        assert!(
            errs.iter().any(|e| e.contains("not verified")),
            "excluded runtime must be reported, got {:?}",
            errs
        );

        // With protection OFF, the arm stays byte-for-byte clean.
        let mut off = runtimes;
        for r in off.iter_mut() {
            r.bitrot_status = crate::storage::erasure_coding::ec_bitrot::BitrotStatus::Off;
            r.bitrot = None;
        }
        let off_refs: Vec<&EcVolume> = off.iter().collect();
        assert_eq!(
            EcChecksumScrubPlan::for_volumes(&off_refs).unwrap().run(),
            (0, Vec::new(), Vec::new()),
            "BitrotOff must stay clean-empty for Go parity"
        );
    }

    /// The Invalid arm returns a non-empty error vector of its own, so the
    /// Go-parity contract that silences the Off arm does not reach it. A volume
    /// with BOTH a malformed sidecar and a fenced-out disk must report BOTH:
    /// reporting only the sidecar hides an unscanned disk behind an unrelated
    /// integrity error, which is the failure mode this whole change exists to
    /// close.
    #[test]
    fn test_checksum_scrub_reports_skips_on_a_malformed_sidecar() {
        use crate::storage::erasure_coding::ec_bitrot::BitrotStatus;

        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let mut runtimes = split_runtimes(dir, VolumeId(1), &[&[0, 1, 2], &[3, 4, 5]]);
        runtimes[0].encode_ts_ns = 100; // excluded: older known identity
        runtimes[1].encode_ts_ns = 200; // anchor
        for r in runtimes.iter_mut() {
            r.bitrot_status = BitrotStatus::Invalid;
        }
        let refs: Vec<&EcVolume> = runtimes.iter().collect();

        let (scanned, broken, errs) = EcChecksumScrubPlan::for_volumes(&refs).unwrap().run();
        assert_eq!(
            scanned, 0,
            "a malformed sidecar must not be scanned against"
        );
        assert!(broken.is_empty(), "must never blame shards: {:?}", broken);
        assert_eq!(
            errs.len(),
            2,
            "expected the sidecar error AND the skip: {:?}",
            errs
        );
        assert!(
            errs[0].contains("malformed/unverifiable"),
            "the sidecar error stays first: {:?}",
            errs
        );
        assert!(
            errs[1].contains("belong to encode run 100") && errs[1].contains("not verified"),
            "the excluded disk must still be named: {:?}",
            errs
        );
    }

    /// When the anchor's sidecar came from a directory belonging to a runtime
    /// the fence excluded, the checksums cannot be trusted against the anchor's
    /// shards. Report that, never "shard N is corrupt".
    #[test]
    fn test_checksum_scrub_reports_unverifiable_sidecar_from_excluded_runtime() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let mut runtimes = split_runtimes(dir, VolumeId(1), &[&[0, 1, 2], &[3, 4, 5]]);

        // Mount actually resolved the real sidecar from `dir` (both subsets
        // share one physical directory in this fixture), confirming
        // `load_bitrot_for_generation` populates the field before the test
        // overwrites it below to simulate a cross-disk borrow.
        assert_eq!(
            runtimes[1].bitrot_source_dir, dir,
            "mount must record the dir the sidecar was actually resolved from"
        );

        // Runtime 0 is an older encode run, and the anchor resolved its sidecar
        // from runtime 0's directory.
        runtimes[0].encode_ts_ns = 100;
        runtimes[0].dir = "/disk-old".to_string();
        runtimes[0].dir_idx = "/disk-old".to_string();
        runtimes[1].encode_ts_ns = 200;
        runtimes[1].dir = "/disk-new".to_string();
        runtimes[1].dir_idx = "/disk-new".to_string();
        runtimes[1].bitrot_source_dir = "/disk-old".to_string();
        let refs: Vec<&EcVolume> = runtimes.iter().collect();

        let (scanned, broken, errs) = EcChecksumScrubPlan::for_volumes(&refs).unwrap().run();
        assert_eq!(
            scanned, 0,
            "an unverifiable sidecar must not be scanned against"
        );
        assert!(broken.is_empty(), "must never blame shards: {:?}", broken);
        assert!(
            errs.iter().any(|e| e.contains("unverifiable")),
            "expected an unverifiable-protection note, got {:?}",
            errs
        );
        assert!(
            errs.iter().any(|e| e.contains("were not verified")),
            "the fenced-out runtime must still be reported alongside the unverifiable note: {:?}",
            errs
        );
    }

    /// The provenance rule fires only when a runtime was actually excluded. A
    /// healthy single-encode volume whose sidecar lives on a sibling disk is the
    /// normal mirrored case and must scrub normally.
    #[test]
    fn test_checksum_scrub_provenance_rule_does_not_fire_without_exclusions() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let mut runtimes = split_runtimes(
            dir,
            VolumeId(1),
            &[&[0, 1, 2, 3, 4, 5, 6], &[7, 8, 9, 10, 11, 12, 13]],
        );
        // Same generation: nothing is excluded, so provenance is irrelevant even
        // though the anchor's sidecar appears to have come from somewhere
        // neither runtime owns. (The anchor is runtimes[0]: `merge_ec_runtimes`
        // picks the first shard-bearing runtime at the anchor generation.)
        runtimes[0].bitrot_source_dir = "/somewhere-else".to_string();
        let refs: Vec<&EcVolume> = runtimes.iter().collect();

        let (scanned, broken, errs) = EcChecksumScrubPlan::for_volumes(&refs).unwrap().run();
        assert!(scanned > 0, "a normal volume must still be scanned");
        assert!(broken.is_empty(), "{:?}", broken);
        assert!(
            !errs.iter().any(|e| e.contains("unverifiable")),
            "{:?}",
            errs
        );
    }

    /// The identity fence keys on `encode_ts_ns` ALONE, never on geometry, so
    /// two same-generation runtimes whose `.vif`s disagree about the layout do
    /// merge -- and `slots` is then sized to the WIDER of them while the
    /// volume's actual shard-id range is the ANCHOR's `data + parity`.
    ///
    /// The two consumers disagreed about exactly that range: the mode 2|5 arm
    /// builds `dirs` over `0..anchor.data_shards + anchor.parity_shards` and
    /// silently drops the surplus slots, while `EcChecksumScrubPlan` iterated
    /// the full width and reported each one "present but missing from sidecar
    /// manifest". Nothing in this volume describes those ids -- the sidecar
    /// manifest and the Reed-Solomon matrix are both the anchor's -- so that
    /// message is the width disagreement talking, not a finding.
    #[test]
    fn test_checksum_scrub_truncates_slots_to_the_anchors_geometry() {
        use crate::pb::volume_server_pb::{
            ChecksumAlgorithm, EcBitrotProtection, EcShardChecksums,
        };
        use crate::storage::erasure_coding::ec_bitrot;
        use crate::storage::volume::{VifEcShardConfig, VifVolumeInfo};

        let tmp = TempDir::new().unwrap();
        let vid = VolumeId(1);

        // One dir per geometry: `EcVolume::new` reads `data_shards`/
        // `parity_shards` from the `.vif` beside the volume, so two runtimes can
        // only disagree if they mount from different directories.
        let seed = |name: &str, ds: u32, ps: u32, shard_id: u8| -> String {
            let dir = tmp.path().join(name);
            std::fs::create_dir_all(&dir).unwrap();
            let d = dir.to_str().unwrap().to_string();
            let base = crate::storage::volume::volume_file_name(&d, "", vid);
            std::fs::write(format!("{}.ecx", base), b"").unwrap();
            std::fs::write(format!("{}.ecj", base), b"").unwrap();
            std::fs::write(
                format!("{}.vif", base),
                serde_json::to_string(&VifVolumeInfo {
                    version: 3,
                    ec_shard_config: Some(VifEcShardConfig {
                        data_shards: ds,
                        parity_shards: ps,
                        encode_ts_ns: 500,
                        ..Default::default()
                    }),
                    ..Default::default()
                })
                .unwrap(),
            )
            .unwrap();
            std::fs::write(
                format!("{}.ec{:02}", base, shard_id),
                b"shard data nonempty",
            )
            .unwrap();
            d
        };

        // 10+4, and the only disk with a sidecar -- so it supplies `prot`, and
        // its manifest covers shard ids 0..=13 and nothing else.
        let narrow = seed("narrow", 10, 4, 0);
        let prot = EcBitrotProtection {
            algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32,
            block_size: ec_bitrot::DEFAULT_BITROT_BLOCK_SIZE as u32,
            generation: 0,
            ec_shard_config: Some(ec_bitrot::ec_shard_config(10, 4, 0)),
            shards: (0..14u32)
                .map(|shard_id| EcShardChecksums {
                    shard_id,
                    covered_size: 4,
                    block_crc32c: vec![0u8; 4],
                })
                .collect(),
            encode_uuid: vec![0u8; 16],
        };
        ec_bitrot::save_bitrot_sidecar(
            &ec_bitrot::bitrot_sidecar_path(
                &crate::storage::volume::volume_file_name(&narrow, "", vid),
                0,
            ),
            &prot,
        )
        .unwrap();

        // 12+4: 16 slots, and it holds shard 14 -- an id the anchor's layout
        // does not contain at all. Same encode_ts_ns, but the geometry fence
        // now excludes it rather than merging incompatible shards.
        let wide = seed("wide", 12, 4, 14);

        let mut narrow_v = EcVolume::new(&narrow, &narrow, "", vid).unwrap();
        narrow_v
            .add_shard(EcVolumeShard::new(&narrow, "", vid, 0))
            .unwrap();
        let mut wide_v = EcVolume::new(&wide, &wide, "", vid).unwrap();
        wide_v
            .add_shard(EcVolumeShard::new(&wide, "", vid, 14))
            .unwrap();

        assert_eq!(narrow_v.shards.len(), 14);
        assert_eq!(wide_v.shards.len(), 16);
        assert_eq!(
            narrow_v.encode_ts_ns, wide_v.encode_ts_ns,
            "the two runtimes must share a generation"
        );
        assert_eq!(
            narrow_v.bitrot_status,
            crate::storage::erasure_coding::ec_bitrot::BitrotStatus::On,
            "the narrow disk must carry usable protection"
        );

        let refs: Vec<&EcVolume> = vec![&narrow_v, &wide_v];
        let merged = merge_ec_runtimes(&refs).unwrap();
        assert!(
            std::ptr::eq(merged.anchor, refs[0]),
            "the NARROW runtime must anchor"
        );
        assert!(
            !merged.skipped.is_empty(),
            "the wide runtime must be excluded by the geometry fence: {:?}",
            merged.skipped
        );
        assert_eq!(
            merged.slots.len(),
            14,
            "slots are sized to the anchor's geometry, not the excluded wide runtime"
        );
        assert!(
            merged.slots.get(14).is_none() || merged.slots.get(14).unwrap().is_none(),
            "shard 14 from the excluded runtime must not appear in slots"
        );

        let (scanned, broken, errs) = EcChecksumScrubPlan::for_volumes(&refs).unwrap().run();
        assert!(
            !errs.iter().any(|e| e.contains("shard 14")),
            "shard 14 belongs to an excluded runtime and must not be scanned: {:?}",
            errs
        );
        assert!(
            scanned > 0,
            "the anchor's shard 0 must still be scanned: {:?}",
            errs
        );
        assert_eq!(
            broken,
            vec![0],
            "shard 0 is inside the anchor's geometry and its bytes do not match \
             the manifest, so it must still be reported: errs={:?}",
            errs
        );
    }

    /// `for_volumes` must reach every runtime's shards, not just the first's.
    /// A needle walk alone can't prove that with this fixture: ~500 bytes of
    /// data under a legacy 1GiB large block puts every needle interval on
    /// shard 0, so the slot vector is asserted directly. The walk-level checks
    /// below additionally show the merged plan verifies cleanly and then
    /// detects corruption reachable through the first runtime, while a
    /// single-runtime plan stays blind to it.
    #[test]
    fn test_scrub_local_for_volumes_merges_slots_across_runtimes() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let runtimes = split_runtimes(dir, VolumeId(1), &[&[0, 1, 2], &[7, 8, 9]]);
        let refs: Vec<&EcVolume> = runtimes.iter().collect();

        let (count, broken, errs) = EcLocalScrubPlan::for_volumes(&refs).unwrap().run();
        assert!(count > 0, "the merged plan walked no needles");
        assert!(
            broken.is_empty(),
            "clean volume reported broken shards: {:?}",
            broken
        );
        assert!(errs.is_empty(), "clean volume reported errors: {:?}", errs);

        // The property this task exists to deliver, asserted directly on the
        // plan rather than through a needle walk. With ~500 bytes of fixture
        // data and a legacy 1GiB large block every needle interval lands in
        // shard 0, so no needle walk can ever reach a sibling runtime's
        // shards -- but the slot vector can. Neither runtime holds every
        // shard here, so the gap at shard 3 makes compaction load-bearing:
        // a compacting implementation could still satisfy a "populated slots
        // are reachable" check while silently reindexing the vector.
        let merged_plan = EcLocalScrubPlan::for_volumes(&refs).unwrap();
        assert_eq!(
            merged_plan.shards.len(),
            14,
            "slots must span the full 10+4 shard space"
        );
        assert!(
            merged_plan.shards[0].is_some(),
            "disk 0's shard 0 must be reachable"
        );
        assert!(
            merged_plan.shards[3].is_none(),
            "a shard no runtime holds must stay an empty slot -- compaction would fill it"
        );
        assert!(
            merged_plan.shards[7].is_some(),
            "disk 1's shard 7 must be reachable -- the bug being fixed"
        );

        // A single-runtime plan still sees only its own disk, which is exactly
        // what made aggregation necessary.
        let solo = EcLocalScrubPlan::for_volumes(&[refs[0]]).unwrap();
        assert!(solo.shards[0].is_some());
        assert!(
            solo.shards[7].is_none(),
            "refs[0] alone must not see the sibling's shard 7"
        );

        // This fixture's few hundred bytes of needle data all land inside
        // shard 0's legacy 1GiB large block (the tiny volume never fills even
        // one), so shard 0 alone — held only by the first runtime — carries
        // every needle. Corrupting a needle body byte there is the LOCAL
        // analogue of Task 3's sibling-disk corruption: the whole point is
        // that a partial-shard `scrub_local` cannot see it at all (it just
        // silently skips a needle it can't reassemble locally — that is not
        // an error), while the merged plan reaches it through refs[0].
        let shard0 = format!("{}/1.ec00", dir);
        let mut bytes = std::fs::read(&shard0).unwrap();
        assert!(
            bytes.len() > 268,
            "fixture shrank below the corruption offset; split_runtimes's \
             needle payloads changed, so this offset needs picking again"
        );
        bytes[268] ^= 0xFF; // inside needle 4's data payload
        std::fs::write(&shard0, &bytes).unwrap();

        let (_, _, merged_errs) = EcLocalScrubPlan::for_volumes(&refs).unwrap().run();
        assert!(
            !merged_errs.is_empty(),
            "merged plan must detect corruption on shard 0, reachable through refs[0]"
        );

        // refs[1] never mounted shard 0, so it cannot see this corruption at
        // all — that silent blind spot is exactly the bug aggregation fixes.
        let (_, _, refs1_errs) = refs[1].scrub_local();
        assert!(
            refs1_errs.is_empty(),
            "runtime without shard 0 has no way to detect this corruption: {:?}",
            refs1_errs
        );
    }

    /// LOCAL's legacy `shard_size` fallback (`dat_file_size == 0`) is now a
    /// NODE-WIDE input: it feeds `locate_data`'s offset math for every merged
    /// sibling's shards, not just the anchor's. `anchor.shard_file_size()`
    /// returns the anchor's FIRST held shard rather than a maximum, so one
    /// truncated shard on the anchor disk would mis-offset every needle read
    /// across every disk and manufacture corruption reports wholesale. Take the
    /// max over the merged slots, the way `verify_ec_shards` already answers the
    /// same question (`if size > shard_size { shard_size = size }`).
    #[test]
    fn test_scrub_local_shard_size_fallback_takes_the_max_across_disks() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();

        // Mount two shardless runtimes first so the shard files can be doctored
        // before `add_shard` caches their sizes.
        let mut runtimes = split_runtimes(dir, VolumeId(1), &[&[], &[]]);

        // Disk 0's shard 0 is truncated -- a partially-copied shard, which is
        // exactly the containment this fallback used to have and now must not
        // lose.
        let shard0_path = format!("{}/1.ec00", dir);
        let full = std::fs::metadata(&shard0_path).unwrap().len();
        assert!(full > 1, "fixture shard is too small to truncate");
        std::fs::OpenOptions::new()
            .write(true)
            .open(&shard0_path)
            .unwrap()
            .set_len(full / 2)
            .unwrap();

        runtimes[0]
            .add_shard(EcVolumeShard::new(dir, "", VolumeId(1), 0))
            .unwrap();
        runtimes[1]
            .add_shard(EcVolumeShard::new(dir, "", VolumeId(1), 1))
            .unwrap();
        let refs: Vec<&EcVolume> = runtimes.iter().collect();

        let intact = std::fs::metadata(format!("{}/1.ec01", dir)).unwrap().len() as i64;
        let truncated = (full / 2) as i64;
        assert!(
            truncated < intact,
            "fixture must leave disk 0's shard SHORTER than disk 1's"
        );

        let merged = merge_ec_runtimes(&refs).unwrap();
        assert!(
            std::ptr::eq(merged.anchor, refs[0]),
            "the truncated disk must anchor, or the old sizing was already right"
        );
        assert_eq!(
            refs[0].dat_file_size, 0,
            "this fixture writes no .vif, so the legacy fallback is the path \
             under test; with a dat_file_size the branch is never reached"
        );
        assert_eq!(
            refs[0].shard_file_size(),
            truncated,
            "the old source -- the anchor's first held shard -- is the \
             truncated one, which is what makes this observable"
        );

        let plan = EcLocalScrubPlan::for_volumes(&refs).unwrap();
        assert_eq!(
            plan.shard_size,
            intact - 1,
            "one disk's truncated shard must not size every merged sibling's \
             shards; locate_data would then mis-offset every needle on every \
             disk and report corruption that is not there"
        );
    }

    /// LOCAL has no bitrot status gate, so an excluded runtime is always
    /// reported.
    #[test]
    fn test_scrub_local_reports_skipped_runtimes() {
        let tmp = TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let mut runtimes = split_runtimes(dir, VolumeId(1), &[&[0, 1, 2], &[3, 4, 5]]);
        runtimes[0].encode_ts_ns = 100;
        runtimes[1].encode_ts_ns = 200;
        let refs: Vec<&EcVolume> = runtimes.iter().collect();

        let (_, _, errs) = EcLocalScrubPlan::for_volumes(&refs).unwrap().run();
        assert!(
            errs.iter().any(|e| e.contains("not verified")),
            "excluded runtime must be reported, got {:?}",
            errs
        );
    }

    /// Mount a bare EC volume: an empty .ecx is enough to exercise the
    /// shard-location cache, which is pure in-memory state.
    fn mount_bare_ec_volume(dir: &str) -> EcVolume {
        let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(1));
        std::fs::write(format!("{}.ecx", base), b"").unwrap();
        EcVolume::new(dir, dir, "", VolumeId(1)).unwrap()
    }

    /// The map and the refresh time it was learned at must become visible in the
    /// same step. A merge that published the map first and stamped the time
    /// afterwards let a reader in between pair a fresh map with the previous
    /// refresh time -- and the freshness heuristic judges exactly that pair, so
    /// the reader re-looked-up a map that had just been refreshed.
    #[test]
    fn merge_shard_locations_publishes_map_and_time_together() {
        let tmp = TempDir::new().unwrap();
        let vol = mount_bare_ec_volume(tmp.path().to_str().unwrap());

        let (locations, refreshed_at) = vol.shard_locations_snapshot();
        assert!(locations.is_empty(), "a fresh mount knows no locations");
        assert!(
            refreshed_at.is_none(),
            "a volume that never refreshed has no refresh time"
        );

        let reply: HashMap<ShardId, Vec<String>> =
            HashMap::from([(0, vec!["a:1".to_string()]), (1, vec!["b:2".to_string()])]);
        let merged = vol.merge_shard_locations(reply.clone());
        assert_eq!(merged, reply, "merge returns the resulting map");

        let (locations, refreshed_at) = vol.shard_locations_snapshot();
        assert_eq!(locations, reply, "the snapshot reports the merged map");
        assert!(
            refreshed_at.is_some(),
            "the merge that produced that map also stamped its refresh time"
        );

        // Upsert, not replace: a reply that omits a cached shard keeps it.
        // `before` is taken outside the merge so the assertion fails if the
        // second merge did not re-stamp: `later >= refreshed_at` would hold on
        // the first merge's stamp alone.
        let before = Instant::now();
        let merged = vol.merge_shard_locations(HashMap::from([(1, vec!["c:3".to_string()])]));
        assert_eq!(merged.get(&0), Some(&vec!["a:1".to_string()]));
        assert_eq!(merged.get(&1), Some(&vec!["c:3".to_string()]));
        let (locations, later) = vol.shard_locations_snapshot();
        assert_eq!(locations, merged, "the snapshot reports the merged map");
        assert!(
            later.unwrap() >= before,
            "the second merge re-stamped the refresh time"
        );
    }

    /// The stale mark is consumed by the refresh that acts on it, exactly once:
    /// otherwise every later read keeps re-looking-up a map no one has disproved
    /// since. A claim that declines the refresh must leave the mark standing.
    #[test]
    fn claim_shard_locations_refresh_consumes_the_stale_mark_once() {
        let tmp = TempDir::new().unwrap();
        let vol = mount_bare_ec_volume(tmp.path().to_str().unwrap());

        assert!(
            !vol.claim_shard_locations_refresh(|stale| stale),
            "an untouched cache carries no mark"
        );

        vol.mark_shard_locations_stale();
        assert!(
            vol.claim_shard_locations_refresh(|stale| stale),
            "the mark is visible to the next claim"
        );
        assert!(
            !vol.claim_shard_locations_refresh(|stale| stale),
            "the claim that acted on the mark consumed it"
        );

        vol.mark_shard_locations_stale();
        assert!(
            !vol.claim_shard_locations_refresh(|_| false),
            "the refresh decision stays the caller's"
        );
        assert!(
            vol.claim_shard_locations_refresh(|stale| stale),
            "a claim that did not refresh leaves the mark for the next one"
        );
    }
}

#[cfg(test)]
mod uniform_layout_tests {
    use super::*;
    use crate::storage::needle_map::NeedleMapKind;
    use crate::storage::volume::{VifEcShardConfig, VifVolumeInfo, Volume, VolumeSpec};
    use tempfile::TempDir;

    // Write ~26MB of needles so the uniform block size (3MB) diverges from the
    // legacy layout, encode, and verify EcVolume reads every needle back
    // through the .vif-recorded geometry. A legacy-encoded fixture (no .vif
    // block size) must keep reading through the legacy interpretation.
    #[test]
    fn test_read_needles_uniform_and_legacy_layouts() {
        for legacy in [false, true] {
            let tmp = TempDir::new().unwrap();
            let dir = tmp.path().to_str().unwrap();
            let vid = VolumeId(8);

            let mut v = Volume::new(
                dir,
                dir,
                vid,
                NeedleMapKind::InMemory,
                &VolumeSpec::default(),
            )
            .unwrap();
            let mut expected: Vec<(NeedleId, Vec<u8>)> = Vec::new();
            for i in 1u64..=11 {
                let size = if i <= 5 { 4 << 20 } else { 1 << 20 };
                let data: Vec<u8> = (0..size)
                    .map(|b| ((b as u64).wrapping_mul(2654435761).wrapping_add(i) >> 8) as u8)
                    .collect();
                let mut n = Needle {
                    id: NeedleId(i),
                    cookie: Cookie(i as u32),
                    data: data.clone(),
                    data_size: data.len() as u32,
                    ..Needle::default()
                };
                v.write_needle(&mut n, true, false).unwrap();
                expected.push((NeedleId(i), data));
            }
            v.sync_to_disk().unwrap();
            let dat_size = v.dat_file_size().unwrap() as i64;
            v.close();

            let block_size = if legacy {
                // Legacy fixture: two-tier encode plus a .vif without a block
                // size, the state every pre-upgrade EC volume is in.
                use crate::storage::erasure_coding::ec_bitrot::{
                    DEFAULT_BITROT_BLOCK_SIZE, ShardChecksumBuilder,
                };
                use reed_solomon_erasure::galois_8::ReedSolomon;
                let base = crate::storage::volume::volume_file_name(dir, "", vid);
                crate::storage::erasure_coding::ec_encoder::write_sorted_ecx_from_idx(
                    &format!("{}.idx", base),
                    &format!("{}.ecx", base),
                )
                .unwrap();
                let dat_file = std::fs::File::open(format!("{}.dat", base)).unwrap();
                let rs = ReedSolomon::new(10, 4).unwrap();
                let mut shards: Vec<EcVolumeShard> = (0..14u8)
                    .map(|i| EcVolumeShard::new(dir, "", vid, i))
                    .collect();
                for shard in &mut shards {
                    shard.create().unwrap();
                }
                let mut builders: Vec<ShardChecksumBuilder> = (0..14)
                    .map(|_| ShardChecksumBuilder::new(DEFAULT_BITROT_BLOCK_SIZE as i64))
                    .collect();
                crate::storage::erasure_coding::ec_encoder::encode_dat_file(
                    &dat_file,
                    dat_size,
                    &rs,
                    &mut shards,
                    &mut builders,
                    crate::storage::erasure_coding::ec_encoder::EcEncodeLayout {
                        data_shards: 10,
                        parity_shards: 4,
                        buffer_size: 256 * 1024,
                        large_block_size: ERASURE_CODING_LARGE_BLOCK_SIZE,
                        small_block_size: ERASURE_CODING_SMALL_BLOCK_SIZE,
                    },
                )
                .unwrap();
                for shard in &mut shards {
                    shard.close();
                }
                0
            } else {
                let bs = crate::storage::erasure_coding::ec_encoder::write_ec_files(
                    dir, dir, "", vid, 10, 4,
                )
                .unwrap();
                assert!(
                    bs > ERASURE_CODING_SMALL_BLOCK_SIZE as i64,
                    "block size {} does not diverge from the legacy layout",
                    bs
                );
                bs
            };

            let base = crate::storage::volume::volume_file_name(dir, "", vid);
            let vif = VifVolumeInfo {
                version: Version::current().0 as u32,
                dat_file_size: dat_size,
                ec_shard_config: Some(VifEcShardConfig {
                    data_shards: 10,
                    parity_shards: 4,
                    block_size,
                    ..Default::default()
                }),
                ..Default::default()
            };
            std::fs::write(
                format!("{}.vif", base),
                serde_json::to_string_pretty(&vif).unwrap(),
            )
            .unwrap();
            std::fs::remove_file(format!("{}.dat", base)).unwrap();
            std::fs::remove_file(format!("{}.idx", base)).unwrap();

            let mut vol = EcVolume::new(dir, dir, "", vid).unwrap();
            assert_eq!(
                vol.block_size, block_size,
                "block size not loaded from .vif"
            );
            for i in 0..10u8 {
                vol.add_shard(EcVolumeShard::new(dir, "", vid, i)).unwrap();
            }

            for (id, data) in &expected {
                let n = vol
                    .read_ec_shard_needle(*id)
                    .unwrap()
                    .unwrap_or_else(|| panic!("needle {} not found (legacy={})", id.0, legacy));
                assert_eq!(
                    n.data, *data,
                    "needle {} data mismatch (legacy={})",
                    id.0, legacy
                );
            }
        }
    }

    // A present-but-malformed .vif must FAIL the mount: every new encode
    // records a positive uniform block size there, and silently defaulting
    // to the legacy layout would serve those shards with the wrong offset
    // math. Absence stays legal — legacy volumes predate the sidecar.
    #[test]
    fn new_fails_on_malformed_vif() {
        let tmp = tempfile::TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(1));
        std::fs::write(format!("{}.ecx", base), b"").unwrap();
        std::fs::write(format!("{}.vif", base), b"not json").unwrap();
        let res = EcVolume::new(dir, dir, "", VolumeId(1));
        assert!(res.is_err(), "mount over a malformed .vif must fail");
    }

    #[test]
    fn new_absent_vif_mounts_with_defaults() {
        let tmp = tempfile::TempDir::new().unwrap();
        let dir = tmp.path().to_str().unwrap();
        let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(1));
        std::fs::write(format!("{}.ecx", base), b"").unwrap();
        let ev = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
        assert_eq!(ev.block_size, 0, "legacy mount must use the legacy layout");
    }

    /// A rebuild lands on one disk, but the volume's metadata may live on
    /// another. Reading only the selected directory returns the default 10+4
    /// and the legacy block layout, which reconstructs a custom-ratio or
    /// uniform volume through the wrong matrix.
    fn seed_uniform_sidecar(base: &str, ds: u32, ps: u32, block: i64) {
        use crate::pb::volume_server_pb::{
            ChecksumAlgorithm, EcBitrotProtection, EcShardChecksums,
        };
        use crate::storage::erasure_coding::ec_bitrot;
        let prot = EcBitrotProtection {
            algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32,
            block_size: ec_bitrot::DEFAULT_BITROT_BLOCK_SIZE as u32,
            generation: 0,
            ec_shard_config: Some(ec_bitrot::ec_shard_config(ds, ps, block)),
            shards: (0..(ds + ps))
                .map(|shard_id| EcShardChecksums {
                    shard_id,
                    covered_size: 4,
                    block_crc32c: vec![0u8; 4],
                })
                .collect(),
            encode_uuid: vec![0u8; 16],
        };
        ec_bitrot::save_bitrot_sidecar(&ec_bitrot::bitrot_sidecar_path(base, 0), &prot).unwrap();
    }

    fn seed_config_free_vif(base: &str) {
        std::fs::write(format!("{}.vif", base), r#"{"version":3,"datFileSize":0}"#).unwrap();
    }

    // A .vif that omits ecShardConfig answers nothing about the layout, so it
    // must not short-circuit the sidecar search. Returning as soon as any
    // parseable vif turned up resolved a 12+4 uniform volume as 10+4 legacy.
    #[test]
    fn read_ec_shard_config_falls_back_when_the_vif_has_no_config() {
        let d = tempfile::TempDir::new().unwrap();
        let dir = d.path().to_str().unwrap();
        let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(11));
        seed_config_free_vif(&base);
        seed_uniform_sidecar(&base, 12, 4, 3 * 1024 * 1024);

        let got = read_ec_shard_config(dir, dir, "", VolumeId(11)).unwrap();
        assert_eq!(got, (12, 4, 3 * 1024 * 1024));
    }

    // Split -dir/-dir.idx: the vif and the sidecar both live with the INDEX,
    // and the resolver was told to report the DATA directory, so the fallback
    // looked in a directory holding neither.
    #[test]
    fn read_ec_shard_config_finds_a_config_free_vifs_sidecar_in_the_index_dir() {
        let data = tempfile::TempDir::new().unwrap();
        let idx = tempfile::TempDir::new().unwrap();
        let (dir, dir_idx) = (data.path().to_str().unwrap(), idx.path().to_str().unwrap());
        let idx_base = crate::storage::volume::volume_file_name(dir_idx, "", VolumeId(12));
        seed_config_free_vif(&idx_base);
        seed_uniform_sidecar(&idx_base, 12, 4, 3 * 1024 * 1024);

        let got = read_ec_shard_config(dir, dir_idx, "", VolumeId(12)).unwrap();
        assert_eq!(got, (12, 4, 3 * 1024 * 1024));
    }

    // Same gap in the cross-disk resolver the rebuild uses.
    #[test]
    fn read_ec_shard_config_across_dirs_falls_back_when_the_vif_has_no_config() {
        let data = tempfile::TempDir::new().unwrap();
        let sib = tempfile::TempDir::new().unwrap();
        let (dir, sibling) = (data.path().to_str().unwrap(), sib.path().to_str().unwrap());
        seed_config_free_vif(&crate::storage::volume::volume_file_name(
            dir,
            "",
            VolumeId(13),
        ));
        seed_uniform_sidecar(
            &crate::storage::volume::volume_file_name(sibling, "", VolumeId(13)),
            12,
            4,
            3 * 1024 * 1024,
        );

        let got =
            read_ec_shard_config_across_dirs(dir, dir, &[sibling.to_string()], "", VolumeId(13))
                .unwrap();
        assert_eq!(got, (12, 4, 3 * 1024 * 1024));
    }

    // Absence stays legal: a volume predating both records is genuinely legacy.
    #[test]
    fn read_ec_shard_config_config_free_vif_without_a_sidecar_is_legacy() {
        let d = tempfile::TempDir::new().unwrap();
        let dir = d.path().to_str().unwrap();
        seed_config_free_vif(&crate::storage::volume::volume_file_name(
            dir,
            "",
            VolumeId(14),
        ));
        let got = read_ec_shard_config(dir, dir, "", VolumeId(14)).unwrap();
        assert_eq!(got, (10, 4, 0));
    }

    #[test]
    fn read_ec_shard_config_finds_a_sibling_disks_vif() {
        let a = tempfile::TempDir::new().unwrap();
        let b = tempfile::TempDir::new().unwrap();
        let (rebuild, sibling) = (a.path().to_str().unwrap(), b.path().to_str().unwrap());
        let base = crate::storage::volume::volume_file_name(sibling, "", VolumeId(3));
        std::fs::write(
            format!("{}.vif", base),
            r#"{"version":3,"ecShardConfig":{"dataShards":12,"parityShards":4,"blockSize":3145728}}"#,
        )
        .unwrap();

        let (ds, ps, bs) = read_ec_shard_config_across_dirs(
            rebuild,
            rebuild,
            &[sibling.to_string()],
            "",
            VolumeId(3),
        )
        .unwrap();
        assert_eq!((ds, ps, bs), (12, 4, 3 * 1024 * 1024));
    }

    /// Same, for the sidecar: with no .vif anywhere it is the surviving record
    /// of the geometry, wherever it sits.
    // A split -dir/-dir.idx location keeps its metadata with the INDEX, and
    // callers leave their own index directory out of other_dirs because it is
    // passed separately. With no .vif anywhere the generation-0 .ecsum is the
    // only record of the layout, so missing that directory resolves a 12+4
    // uniform volume to 10+4 legacy and reconstructs through the wrong matrix.
    #[test]
    fn read_ec_shard_config_finds_the_sidecar_in_its_own_index_dir() {
        use crate::pb::volume_server_pb::{
            ChecksumAlgorithm, EcBitrotProtection, EcShardChecksums,
        };
        use crate::storage::erasure_coding::ec_bitrot;

        let data = tempfile::TempDir::new().unwrap();
        let idx = tempfile::TempDir::new().unwrap();
        let (dir, dir_idx) = (data.path().to_str().unwrap(), idx.path().to_str().unwrap());
        let base = crate::storage::volume::volume_file_name(dir_idx, "", VolumeId(7));
        let prot = EcBitrotProtection {
            algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32,
            block_size: ec_bitrot::DEFAULT_BITROT_BLOCK_SIZE as u32,
            generation: 0,
            ec_shard_config: Some(ec_bitrot::ec_shard_config(12, 4, 3 * 1024 * 1024)),
            shards: (0..16u32)
                .map(|shard_id| EcShardChecksums {
                    shard_id,
                    covered_size: 4,
                    block_crc32c: vec![0u8; 4],
                })
                .collect(),
            encode_uuid: vec![0u8; 16],
        };
        ec_bitrot::save_bitrot_sidecar(&ec_bitrot::bitrot_sidecar_path(&base, 0), &prot).unwrap();

        let (ds, ps, bs) =
            read_ec_shard_config_across_dirs(dir, dir_idx, &[], "", VolumeId(7)).unwrap();
        assert_eq!((ds, ps, bs), (12, 4, 3 * 1024 * 1024));
    }

    #[test]
    fn read_ec_shard_config_finds_a_sibling_disks_sidecar() {
        use crate::pb::volume_server_pb::{
            ChecksumAlgorithm, EcBitrotProtection, EcShardChecksums,
        };
        use crate::storage::erasure_coding::ec_bitrot;

        let a = tempfile::TempDir::new().unwrap();
        let b = tempfile::TempDir::new().unwrap();
        let (rebuild, sibling) = (a.path().to_str().unwrap(), b.path().to_str().unwrap());
        let base = crate::storage::volume::volume_file_name(sibling, "", VolumeId(4));
        let prot = EcBitrotProtection {
            algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32,
            block_size: ec_bitrot::DEFAULT_BITROT_BLOCK_SIZE as u32,
            generation: 0,
            ec_shard_config: Some(ec_bitrot::ec_shard_config(12, 4, 3 * 1024 * 1024)),
            shards: (0..16u32)
                .map(|shard_id| EcShardChecksums {
                    shard_id,
                    covered_size: 4,
                    block_crc32c: vec![0u8; 4],
                })
                .collect(),
            encode_uuid: vec![0u8; 16],
        };
        ec_bitrot::save_bitrot_sidecar(&ec_bitrot::bitrot_sidecar_path(&base, 0), &prot).unwrap();

        let (ds, ps, bs) = read_ec_shard_config_across_dirs(
            rebuild,
            rebuild,
            &[sibling.to_string()],
            "",
            VolumeId(4),
        )
        .unwrap();
        assert_eq!((ds, ps, bs), (12, 4, 3 * 1024 * 1024));
    }
}

/// One volume id's per-disk runtimes resolved into a single scrubbable view.
/// Every scrub mode builds from this, so there is exactly one answer to
/// "which disks count" per volume.
pub(crate) struct MergedEcRuntimes<'a> {
    /// Volume-level metadata source: geometry, `.ecx` handles, version. NOT
    /// bitrot protection — the `.ecsum` sidecar is per-DISK state, so
    /// `EcChecksumScrubPlan::for_volumes` sources it from the first `merged`
    /// runtime that has any.
    pub anchor: &'a EcVolume,
    /// Runtimes whose shards are safe to verify together.
    pub merged: Vec<&'a EcVolume>,
    /// Indexed BY SHARD ID. `None` is a shard no merged runtime holds. The
    /// volume's shard-id range is the anchor's geometry
    /// (`0..data_shards + parity_shards`); consumers truncate to it.
    pub slots: Vec<Option<(&'a EcVolume, &'a EcVolumeShard)>>,
    /// One line per runtime excluded by the identity or geometry fence.
    /// Reported, never dropped.
    pub skipped: Vec<String>,
}

/// Resolve `runtimes` (one vid's mounts, in location order) into a merged view.
/// `None` for an empty slice (the vanished-volume case).
///
/// Two fences admit a runtime into `merged`: the anchor's `encode_ts_ns` (the
/// maximum, so `0` implies every runtime is `0` and nothing is excluded —
/// legacy leniency), and the anchor's geometry (`data_shards`, `parity_shards`,
/// `block_size`). Equal timestamps do not guarantee equal layouts, so a
/// same-generation runtime whose `.vif` disagrees is excluded and reported
/// rather than merged into a plan that would apply the anchor's offsets and
/// checksums to incompatible shards.
pub(crate) fn merge_ec_runtimes<'a>(runtimes: &[&'a EcVolume]) -> Option<MergedEcRuntimes<'a>> {
    let anchor_gen = runtimes.iter().map(|v| v.encode_ts_ns).max()?;

    let gen_matches: Vec<&'a EcVolume> = runtimes
        .iter()
        .copied()
        .filter(|v| v.encode_ts_ns == anchor_gen)
        .collect();

    let anchor = gen_matches
        .iter()
        .copied()
        .find(|v| v.shards.iter().any(|s| s.is_some()))
        .unwrap_or(gen_matches[0]);

    let merged: Vec<&'a EcVolume> = gen_matches
        .iter()
        .copied()
        .filter(|v| {
            v.data_shards == anchor.data_shards
                && v.parity_shards == anchor.parity_shards
                && v.block_size == anchor.block_size
        })
        .collect();

    let width = merged.iter().map(|v| v.shards.len()).max().unwrap_or(0);
    let mut slots: Vec<Option<(&'a EcVolume, &'a EcVolumeShard)>> = vec![None; width];
    for v in &merged {
        for (id, slot) in v.shards.iter().enumerate() {
            if let Some(shard) = slot.as_ref()
                && slots[id].is_none()
            {
                slots[id] = Some((*v, shard));
            }
        }
    }

    let skipped: Vec<String> = runtimes
        .iter()
        .enumerate()
        .filter(|(_, v)| {
            v.encode_ts_ns != anchor_gen
                || v.data_shards != anchor.data_shards
                || v.parity_shards != anchor.parity_shards
                || v.block_size != anchor.block_size
        })
        .map(|(pos, v)| {
            if v.encode_ts_ns != anchor_gen {
                format!(
                    "EC volume {} shards at {} (position {}) belong to encode run {} but the scrub anchors on {}; they were not verified",
                    v.volume_id.0, v.dir, pos, v.encode_ts_ns, anchor_gen
                )
            } else {
                format!(
                    "EC volume {} shards at {} (position {}) share encode run {} but disagree on geometry ({}+{} bs {} vs {}+{} bs {}); they were not verified",
                    v.volume_id.0, v.dir, pos, v.encode_ts_ns,
                    v.data_shards, v.parity_shards, v.block_size,
                    anchor.data_shards, anchor.parity_shards, anchor.block_size
                )
            }
        })
        .collect();

    Some(MergedEcRuntimes {
        anchor,
        merged,
        slots,
        skipped,
    })
}

/// Self-contained input for an EC checksum scrub: the sidecar plus a duplicate
/// of every mounted local shard handle, taken under the store read guard.
///
/// Two things follow from holding descriptors rather than paths. `run()` needs
/// no store access, so the guard is released before a scan that reads every
/// byte of every local shard — under the guard that scan parks the periodic
/// heartbeat's `store.write()`, and `std::sync::RwLock` is write-preferring, so
/// every later reader queues behind it and the node stops serving. And a
/// teardown that legitimately unlinks the shards mid-scan (the heartbeat's
/// `delete_expired_ec_volumes`, `volume_ec_shards_delete`) cannot masquerade as
/// bitrot, because the descriptor outlives the name. Go reads the same way:
/// `ChecksumScrub` goes through `shard.ReadAt`.
///
/// Not `Clone`: it owns those descriptors.
#[derive(Debug)]
pub struct EcChecksumScrubPlan {
    pub volume_id: VolumeId,
    pub prot: Option<crate::pb::volume_server_pb::EcBitrotProtection>,
    pub status: crate::storage::erasure_coding::ec_bitrot::BitrotStatus,
    pub parity_shards: u32,
    /// One entry per LOCAL shard: its id and a duplicate of the mounted
    /// handle (or the error to report). Private so a plan can only be built
    /// (via `checksum_scrub_plan()` or `for_volumes()`) from `&EcVolume`
    /// references, which cannot be obtained without the store guard — which
    /// is what makes "captured under the guard" an invariant rather than a
    /// convention.
    ///
    /// `dup` shares the kernel file offset, so every read here is positional.
    shards: Vec<(u32, std::io::Result<File>)>,
    /// Runtimes the identity/geometry fence excluded, one line each. Surfaced
    /// through `run()`'s errors on the `On`/`Invalid` paths.
    skipped: Vec<String>,
    /// `Some(source_dir)` when the sidecar that supplied `prot`/`status` was
    /// resolved from a directory belonging to a runtime the fence excluded.
    /// `run()` reports unverifiable protection instead of blaming shards.
    unverifiable_sidecar: Option<String>,
    /// One line per merged runtime whose sidecar resolved `Invalid`. Reported
    /// even when protection is taken from a sibling that is `On`, so a
    /// malformed sidecar is never silently discarded.
    invalid_sidecar_errors: Vec<String>,
}

impl EcChecksumScrubPlan {
    /// Build one plan over every per-disk runtime of a volume id. `None` only
    /// when `runtimes` is empty (the vanished-volume case).
    pub fn for_volumes(runtimes: &[&EcVolume]) -> Option<Self> {
        use crate::storage::erasure_coding::ec_bitrot::BitrotStatus;

        let merged = merge_ec_runtimes(runtimes)?;
        let anchor = merged.anchor;

        // Protection is per-DISK state: the `.ecsum` sidecar is not mirrored,
        // and at mount each runtime resolves it from its own directories only,
        // so after a restart the runtime without the sidecar mounts `Off`.
        // Take the first merged runtime that has protection: `On` if any, else
        // `Invalid`, else the anchor's `Off`. This only moves toward more
        // verification (`Off->On`, `Off->Invalid`, `Invalid->On`), never toward
        // silence — the anchor is itself in `merged`, so the fallback is `Off`
        // only when every merged runtime is `Off`.
        let protection_source = merged
            .merged
            .iter()
            .copied()
            .find(|v| matches!(v.bitrot_protection().1, BitrotStatus::On))
            .or_else(|| {
                merged
                    .merged
                    .iter()
                    .copied()
                    .find(|v| matches!(v.bitrot_protection().1, BitrotStatus::Invalid))
            })
            .unwrap_or(anchor);
        let (prot, status) = protection_source.bitrot_protection();

        // Collect every merged runtime whose sidecar resolved Invalid, excluding
        // the protection_source (whose error the Invalid arm of run() already
        // reports). These surface when protection is taken from a sibling that
        // is On, so a malformed sidecar is never silently discarded.
        let invalid_sidecar_errors: Vec<String> = merged
            .merged
            .iter()
            .filter(|v| {
                let v_ptr: *const EcVolume = **v;
                let src_ptr: *const EcVolume = protection_source;
                !std::ptr::eq(v_ptr, src_ptr)
                    && matches!(v.bitrot_protection().1, BitrotStatus::Invalid)
            })
            .map(|v| {
                format!(
                    "EC volume {} bitrot sidecar at {} is malformed/unverifiable (sidecar integrity)",
                    v.volume_id.0, v.dir
                )
            })
            .collect();

        let total = (anchor.data_shards + anchor.parity_shards) as usize;

        let shards = match status {
            crate::storage::erasure_coding::ec_bitrot::BitrotStatus::On => merged
                .slots
                .iter()
                .take(total)
                .enumerate()
                .filter_map(|(id, slot)| slot.map(|(_, shard)| (id as u32, shard.try_clone_file())))
                .collect(),
            _ => Vec::new(),
        };

        // If the fence excluded a runtime AND the sidecar that supplied
        // protection was resolved from that excluded runtime's directory, the
        // checksums cannot be trusted against the merged shards.
        let unverifiable_sidecar = if merged.skipped.is_empty() {
            None
        } else {
            let own_dirs: Vec<String> = merged
                .merged
                .iter()
                .flat_map(|v| [v.dir.as_str(), v.dir_idx.as_str()])
                .map(|d| d.trim_end_matches('/').to_string())
                .collect();
            let src = protection_source.bitrot_source_dir.trim_end_matches('/');
            if src.is_empty() || own_dirs.iter().any(|d| d == src) {
                None
            } else {
                Some(protection_source.bitrot_source_dir.clone())
            }
        };

        Some(EcChecksumScrubPlan {
            volume_id: anchor.volume_id,
            prot,
            status,
            parity_shards: anchor.parity_shards,
            shards,
            skipped: merged.skipped,
            unverifiable_sidecar,
            invalid_sidecar_errors,
        })
    }

    /// The byte-verification pass. Touches only the filesystem — no store, no
    /// lock — so it is safe to hand to `spawn_blocking`.
    pub fn run(self) -> (u64, Vec<u32>, Vec<String>) {
        use crate::storage::erasure_coding::ec_bitrot;
        use crate::storage::erasure_coding::ec_bitrot::BitrotStatus;

        let mut errors: Vec<String> = Vec::new();

        // Mirrors Go's `ChecksumScrub` (`prot, status := ecv.BitrotProtection()`):
        //   - Off: sidecars are optional; return clean (Go: `case BitrotOff: return 0, nil, nil`).
        //   - Invalid: sidecar is present but malformed; report an integrity error.
        //   - On: scan local shards against it.
        let prot = match (self.prot, self.status) {
            (_, BitrotStatus::Off) => {
                return (0, Vec::new(), Vec::new());
            }
            (_, BitrotStatus::Invalid) => {
                let mut errs = vec![format!(
                    "EC volume {} bitrot sidecar is malformed/unverifiable (sidecar integrity)",
                    self.volume_id.0
                )];
                errs.extend(self.skipped);
                errs.extend(self.invalid_sidecar_errors);
                return (0, Vec::new(), errs);
            }
            (Some(p), BitrotStatus::On) => p,
            (None, BitrotStatus::On) => {
                return (0, Vec::new(), Vec::new());
            }
        };

        errors.extend(self.skipped);
        errors.extend(self.invalid_sidecar_errors);

        if let Some(src) = self.unverifiable_sidecar {
            errors.push(format!(
                "EC volume {} bitrot sidecar was resolved from {}, which belongs to an excluded encode run; protection unverifiable",
                self.volume_id.0, src
            ));
            return (0, Vec::new(), errors);
        }

        let block_size = prot.block_size as i64;

        let mut blocks_scanned: u64 = 0;
        let mut mismatched_shards: Vec<u32> = Vec::new();
        // Track shards whose blocks ALL mismatch (wholesale) to detect a
        // stale/wrong sidecar.
        let mut wholesale_mismatch = 0usize;

        for (shard_id, handle) in self.shards {
            let Some(entry) = ec_bitrot::shard_checksums(&prot, shard_id) else {
                errors.push(format!(
                    "EC volume {} shard {} present but missing from sidecar manifest",
                    self.volume_id.0, shard_id
                ));
                continue;
            };

            // The handle was opened under the store lock; reading through it
            // means a concurrent unmount/unlink cannot masquerade as bitrot.
            let file = match &handle {
                Ok(f) => f,
                Err(e) => {
                    errors.push(format!(
                        "EC volume {} shard {} scrub read error: {}",
                        self.volume_id.0, shard_id, e
                    ));
                    continue;
                }
            };

            let expected_blocks = entry.block_crc32c.len() / 4;
            match ec_bitrot::verify_shard_blocks(file, entry, block_size) {
                Ok(mismatched) => {
                    blocks_scanned += expected_blocks as u64;
                    if !mismatched.is_empty() {
                        mismatched_shards.push(shard_id);
                        if expected_blocks > 0 && mismatched.len() == expected_blocks {
                            wholesale_mismatch += 1;
                        }
                    }
                }
                Err(e) => {
                    errors.push(format!(
                        "EC volume {} shard {} scrub read error: {}",
                        self.volume_id.0, shard_id, e
                    ));
                }
            }
        }

        // If more shards mismatch wholesale than parity can mask, the sidecar
        // itself is the likely culprit (stale generation / wrong volume), so
        // suppress the shard-corruption verdict and flag a sidecar-integrity
        // issue instead.
        if wholesale_mismatch > self.parity_shards as usize {
            errors.push(format!(
                "EC volume {}: {} shards mismatch wholesale (> {} parity); suspect stale/wrong sidecar, not shard corruption",
                self.volume_id.0, wholesale_mismatch, self.parity_shards
            ));
            mismatched_shards.clear();
        }

        mismatched_shards.sort_unstable();
        (blocks_scanned, mismatched_shards, errors)
    }
}

/// Self-contained input for an EC index scrub.
///
/// Not `Clone`: it owns the .ecx handle opened under the store lock.
#[derive(Debug)]
pub struct EcIndexScrubPlan {
    pub volume_id: VolumeId,
    pub has_ecx_file: bool,
    pub ecx_path: String,
    pub ecx_file_size: i64,
    pub version: Version,
    /// The .ecx handle opened while the store lock was held. Private, so the
    /// plan can only come from `EcVolume::scrub_index_plan`.
    ecx_handle: std::io::Result<File>,
}

impl EcIndexScrubPlan {
    /// Structural walk of the .ecx index. Filesystem only — no store, no lock.
    pub fn run(self) -> (u64, Vec<String>) {
        if !self.has_ecx_file {
            return (
                0,
                vec![format!(
                    "no ECX file associated with EC volume {}",
                    self.volume_id.0
                )],
            );
        }
        if self.ecx_file_size == 0 {
            return (
                0,
                vec![format!(
                    "zero-size ECX file for EC volume {}",
                    self.volume_id.0
                )],
            );
        }

        // A private fd, so the structural scan never moves the shared ecx_file
        // cursor (the cached handle is read positionally elsewhere). Checked
        // after the two guards above so the error ordering is unchanged.
        let mut ecx_file = match self.ecx_handle {
            Ok(f) => f,
            Err(e) => return (0, vec![format!("open ECX file {}: {}", self.ecx_path, e)]),
        };
        crate::storage::idx::check_index_file(&mut ecx_file, self.ecx_file_size, self.version)
    }
}

/// One local shard as `EcLocalScrubPlan` sees it: the mounted descriptor, the
/// size the scan compares against, and the identity a broken-shard report needs.
#[derive(Debug)]
pub struct EcLocalShard {
    file: std::io::Result<File>,
    /// The shard's cached size, as `scrub_local` has always compared against —
    /// deliberately not a live `metadata()` call in `run()`.
    file_size: i64,
    info: crate::pb::volume_server_pb::EcShardInfo,
}

impl EcLocalShard {
    /// Positional read through the duplicated handle. `dup` shares the kernel
    /// offset with the mounted shard, so this must never seek.
    fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result<usize> {
        let file = self
            .file
            .as_ref()
            .map_err(|e| io::Error::new(e.kind(), e.to_string()))?;
        crate::storage::io::read_full_at(file, buf, offset)
    }
}

/// Self-contained input for an EC LOCAL scrub: the index snapshot, a private
/// .ecx descriptor for the needle walk, and the mounted local shard handles.
///
/// Same reasoning as `EcChecksumScrubPlan` — `scrub_local` reads every local
/// needle's bytes, so it must not run under the store read guard.
///
/// Not `Clone`: it owns descriptors.
#[derive(Debug)]
pub struct EcLocalScrubPlan {
    volume_id: VolumeId,
    version: Version,
    data_shards: u32,
    shard_size: i64,
    large_block_size: i64,
    small_block_size: i64,
    index: EcIndexScrubPlan,
    ecx_path: String,
    ecx_walk: std::io::Result<File>,
    /// Indexed BY SHARD ID; `None` is a shard no merged runtime holds.
    shards: Vec<Option<EcLocalShard>>,
    /// Runtimes the identity fence excluded, one line each. LOCAL has no status
    /// gate, so `run()` always reports these.
    skipped: Vec<String>,
}

impl EcLocalScrubPlan {
    /// Build one plan over every per-disk runtime of a volume id. `None` only
    /// when `runtimes` is empty.
    ///
    /// The shard vector keeps its SLOT structure — index is the shard id, gaps
    /// are shards no merged runtime holds. `run()` reads that distinction to
    /// decide whether a needle can be reassembled locally at all, so compacting
    /// it would silently change which needles get verified.
    pub fn for_volumes(runtimes: &[&EcVolume]) -> Option<Self> {
        let merged = merge_ec_runtimes(runtimes)?;
        let anchor = merged.anchor;

        Some(EcLocalScrubPlan {
            volume_id: anchor.volume_id,
            version: anchor.version,
            data_shards: anchor.data_shards,
            // locate_data wants shardSize = datFileSize / DataShards when known,
            // else ecdFileSize - 1 (shards are padded to the small block size;
            // the -1 avoids an off-by-one in the large-block row count).
            //
            // The fallback takes the MAX over every merged slot, not
            // `anchor.shard_file_size()` -- which returns the anchor's FIRST
            // held shard, not a maximum. Before aggregation the plan only read
            // the anchor's own shards, so a truncated shard there mis-sized only
            // its own runtime; now one disk's truncated shard would set
            // `shard_size` for every merged sibling's shards, mis-offset
            // `locate_data` and manufacture needle corruption across the whole
            // node. `verify_ec_shards` already answers this same question the
            // same way (`if size > shard_size { shard_size = size }`), so this
            // is the in-tree convention rather than a preference. Reached only
            // on the legacy `dat_file_size == 0` path.
            //
            // `.take(..)` honors the `slots` contract: the volume's shard-id
            // range is the ANCHOR's geometry, and a same-generation runtime with
            // a disagreeing `.vif` can populate slots beyond it. Scanning those
            // would let an OUT-OF-GEOMETRY shard -- one nothing in this volume's
            // layout describes -- set the offset math for every shard that IS in
            // it, which is a narrower path to exactly the node-wide mis-sizing
            // this max exists to close.
            shard_size: if anchor.dat_file_size > 0 {
                anchor.dat_file_size / anchor.data_shards as i64
            } else {
                merged
                    .slots
                    .iter()
                    .take((anchor.data_shards + anchor.parity_shards) as usize)
                    .flatten()
                    .map(|(_, s)| s.file_size())
                    .max()
                    .unwrap_or(0)
                    - 1
            },
            large_block_size: anchor.large_block_size(),
            small_block_size: anchor.small_block_size(),
            index: anchor.scrub_index_plan(),
            ecx_path: anchor.ecx_file_name(),
            // A second descriptor: the index plan's is consumed by its own walk,
            // and both seek.
            ecx_walk: File::open(anchor.ecx_file_name()),
            shards: merged
                .slots
                .iter()
                .map(|slot| {
                    slot.map(|(_, s)| EcLocalShard {
                        file: s.try_clone_file(),
                        file_size: s.file_size(),
                        info: s.to_ec_shard_info(),
                    })
                })
                .collect(),
            skipped: merged.skipped,
        })
    }

    /// The needle walk. Filesystem only — no store, no lock.
    pub fn run(
        self,
    ) -> (
        u64,
        Vec<crate::pb::volume_server_pb::EcShardInfo>,
        Vec<String>,
    ) {
        let EcLocalScrubPlan {
            volume_id,
            version,
            data_shards,
            shard_size,
            large_block_size,
            small_block_size,
            index,
            ecx_path,
            ecx_walk,
            shards,
            skipped,
        } = self;

        // Local scan also verifies the index.
        let (_, mut errs) = index.run();

        // LOCAL has no protection status to gate on, so an excluded runtime is
        // always reported rather than silently unscanned.
        errs.extend(skipped);

        let mut broken_shards: HashSet<ShardId> = HashSet::new();
        let mut count: u64 = 0;

        let mut ecx_file = match ecx_walk {
            Ok(f) => f,
            Err(e) => {
                errs.push(format!("open ECX file {}: {}", ecx_path, e));
                return (count, Vec::new(), errs);
            }
        };

        // Reused across every needle/chunk to avoid a per-chunk allocation.
        let mut chunk_buf: Vec<u8> = Vec::new();
        let walk = crate::storage::idx::walk_index_file(&mut ecx_file, 0, |id, offset, size| {
            count += 1;
            if size.is_tombstone() {
                return Ok(());
            }

            // Go recomputes this at the size check below; hoisted because Rust
            // also sizes the reassembly buffer from it. Any negative size other
            // than the -1 tombstone skipped above drives it negative.
            let want = get_actual_size(size, version);

            let locations = ec_locate::locate_data(
                offset.to_actual_offset(),
                Size(want as i32),
                shard_size,
                data_shards,
                large_block_size,
                small_block_size,
            );
            // A needle is verifiable locally only if every shard it spans is local;
            // when any is remote, skip the reassembly buffer entirely.
            let has_remote_chunks = locations.iter().any(|iv| {
                let (sid, _) =
                    iv.to_shard_id_and_offset(data_shards, large_block_size, small_block_size);
                shards.get(sid as usize).and_then(|s| s.as_ref()).is_none()
            });
            let mut read: i64 = 0;
            // `want <= 0` means the row is malformed. Go pays nothing for it:
            // it appends to a nil slice and has no capacity hint here. Rust's
            // per-needle buffer does, and `Vec::with_capacity(negative as usize)`
            // aborts the process -- inside spawn_blocking that surfaces as a
            // JoinError and takes the whole node-wide scrub RPC with it. Fall
            // through with an empty buffer instead: locate_data already returns
            // no intervals for a non-positive size, so read stays 0 and the
            // `read != want` check below reports the row, exactly as Go does.
            let mut data: Vec<u8> = if has_remote_chunks || want <= 0 {
                Vec::new()
            } else {
                Vec::with_capacity(want as usize)
            };
            let mut local_shard_ids: Vec<ShardId> = Vec::new();

            for (i, iv) in locations.iter().enumerate() {
                let (sid, soffset) =
                    iv.to_shard_id_and_offset(data_shards, large_block_size, small_block_size);
                let ssize = iv.size;
                let shard = match shards.get(sid as usize).and_then(|s| s.as_ref()) {
                    Some(s) => s,
                    None => {
                        // Shard is not local; we can't verify it without decoding.
                        read += ssize;
                        continue;
                    }
                };
                local_shard_ids.push(sid);

                if soffset + ssize > shard.file_size {
                    broken_shards.insert(sid);
                    errs.push(format!(
                        "local shard {} for needle {} is too short ({}), cannot read chunk {}/{}",
                        sid,
                        id.0,
                        shard.file_size,
                        i + 1,
                        locations.len()
                    ));
                    continue;
                }

                chunk_buf.resize(ssize as usize, 0);
                match shard.read_at(&mut chunk_buf, soffset as u64) {
                    Err(e) => {
                        broken_shards.insert(sid);
                        errs.push(format!(
                            "failed to read chunk {}/{} for needle {} from local shard {} at offset {}: {}",
                            i + 1,
                            locations.len(),
                            id.0,
                            sid,
                            soffset,
                            e
                        ));
                        continue;
                    }
                    Ok(got) if got as i64 != ssize => {
                        broken_shards.insert(sid);
                        errs.push(format!(
                            "expected {} bytes for chunk {}/{} for needle {} from local shard {}, got {}",
                            ssize,
                            i + 1,
                            locations.len(),
                            id.0,
                            sid,
                            got
                        ));
                        continue;
                    }
                    Ok(_) => {}
                }

                if !has_remote_chunks {
                    data.extend_from_slice(&chunk_buf);
                }
                read += ssize;
            }

            local_shard_ids.sort_unstable();

            if read != want {
                // Like Go, returning from the walk callback aborts the scan.
                return Err(io::Error::other(format!(
                    "expected {} bytes for needle {} on volume {}, got {}",
                    want, id.0, volume_id.0, read
                )));
            }

            // Only a fully-local needle can be reassembled and CRC-checked.
            if !has_remote_chunks {
                let mut n = Needle::default();
                if let Err(e) = n.read_bytes(&data, 0, size, version) {
                    // A delete-state disagreement between the .ecx index and the reassembled
                    // on-disk header (live index vs zero header size) is not corruption.
                    let delete_state_disagrees = matches!(
                        &e,
                        NeedleError::SizeMismatch { found, .. } if size.is_deleted() != (found.0 == 0)
                    );
                    if !delete_state_disagrees {
                        errs.push(format!(
                            "needle {} on volume {}, shards {:?}: {}",
                            id.0, volume_id.0, local_shard_ids, e
                        ));
                    }
                }
            }
            Ok(())
        });
        if let Err(e) = walk {
            // Go appends the walk/callback error verbatim.
            errs.push(e.to_string());
        }

        let mut broken: Vec<crate::pb::volume_server_pb::EcShardInfo> = broken_shards
            .iter()
            .filter_map(|sid| shards.get(*sid as usize).and_then(|s| s.as_ref()))
            .map(|s| s.info.clone())
            .collect();
        broken.sort_by_key(|a| a.shard_id);

        (count, broken, errs)
    }
}