volume: count EC intervals served by reconstruction (#12392)

* volume: count EC reads that fall back to reconstruction

When a needle read can't read an EC shard interval directly (the shard isn't
mounted here and no holder answers), both volume servers rebuild it from the
other shards and the read still succeeds, with nothing to show for it.

Count those fallbacks in SeaweedFS_volumeServer_ec_degraded_read_total{result}
(success, failure), one per interval, at the one place each server falls back:
Go readOneEcShardInterval, Rust fetch_one_interval. Corruption repair and scrub
aren't counted. A line binding each one to the needle and the volume's encode
generation is logged at V(3) in Go and debug in Rust.

* volume: name the EC counter for what it counts

Reed-Solomon reconstruction can't tell corrupt inputs from good ones, and the
needle is validated later, so "success" only ever meant the reconstruction
produced data. Rename the counter to
SeaweedFS_volumeServer_ec_reconstructed_interval_total and the log line to
"ec reconstructed interval", say so in the help text, and pin the behaviour
with a test: an interval rebuilt from corrupt parity counts success while the
read itself fails.

* storage: restore stderr when the log-line test stops early

A fixture failure in readEcNeedle ends the test through t.Fatalf, which
skipped restoring os.Stderr and closing the pipe, so later tests wrote
into a pipe nobody read.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

seaweed-volume/src/server/handlers.rs

@@ -6227,6 +6227,171 @@ mod tests {
         })
     }
 
+    /// EC reconstructed-interval counter: one fresh volume per case, the needle (on
+    /// shard 0) EC-encoded under a known generation, the `missing` shards
+    /// unmounted and the shards in `corrupt` overwritten where the needle lies.
+    /// Returns the read's bytes and the marker lines logged at debug.
+    async fn read_ec_reconstructed(
+        vid: u32,
+        missing: &[crate::storage::erasure_coding::ec_shard::ShardId],
+        corrupt: &[crate::storage::erasure_coding::ec_shard::ShardId],
+    ) -> (Option<Vec<u8>>, Vec<String>) {
+        use crate::storage::erasure_coding::ec_encoder::write_ec_files;
+        use crate::storage::needle_map::NeedleMapKind;
+        use crate::storage::volume::{VifEcShardConfig, VifVolumeInfo, Volume, VolumeSpec};
+        use std::io::{Seek, SeekFrom, Write};
+        let tmp = tempfile::TempDir::new().unwrap();
+        let state = volume_test_state(&tmp);
+        let dir = tmp.path().to_str().unwrap();
+        let v = Volume::new(
+            dir,
+            dir,
+            VolumeId(vid),
+            NeedleMapKind::InMemory,
+            &VolumeSpec::default(),
+        )
+        .unwrap();
+        let mut n = Needle {
+            id: NeedleId(7),
+            cookie: Cookie(TEST_COOKIE),
+            data: b"ec degraded read".to_vec(),
+            data_size: 16,
+            ..Needle::default()
+        };
+        v.write_needle(&mut n, true, false).unwrap();
+        v.sync_to_disk().unwrap();
+        v.close();
+        let block_size = write_ec_files(dir, dir, "", VolumeId(vid), 10, 4).unwrap();
+        let vif = format!("{dir}/{vid}.vif");
+        let mut info: VifVolumeInfo =
+            serde_json::from_str(&std::fs::read_to_string(&vif).unwrap()).unwrap();
+        info.ec_shard_config = Some(VifEcShardConfig {
+            data_shards: 10,
+            parity_shards: 4,
+            encode_ts_ns: EC_GENERATION,
+            block_size,
+        });
+        crate::storage::store::save_vif_volume_info(&vif, &info).unwrap();
+        for shard in corrupt {
+            let mut f = std::fs::OpenOptions::new()
+                .write(true)
+                .open(format!("{dir}/{vid}.ec{shard:02}"))
+                .unwrap();
+            f.seek(SeekFrom::Start(0)).unwrap();
+            f.write_all(&[0xA5; 256]).unwrap();
+        }
+        let mounted: Vec<_> = (0..14).filter(|s| !missing.contains(s)).collect();
+        state
+            .store
+            .write()
+            .unwrap()
+            .mount_ec_shards(VolumeId(vid), "", &mounted)
+            .unwrap();
+
+        let out = Captured::default();
+        let subscriber = tracing_subscriber::fmt()
+            .with_max_level(tracing::Level::DEBUG)
+            .with_ansi(false)
+            .with_writer(out.clone())
+            .finish();
+        let guard = tracing::subscriber::set_default(subscriber);
+        // Another test may have reached the marker's callsite first, with no
+        // subscriber, and cached it as never enabled.
+        tracing::callsite::rebuild_interest_cache();
+        let read = crate::server::store_ec::read_ec_shard_needle_or_miss(
+            &state,
+            VolumeId(vid),
+            NeedleId(7),
+        )
+        .await;
+        drop(guard);
+        let text = String::from_utf8(out.0.lock().unwrap().clone()).unwrap();
+        let lines = text
+            .lines()
+            .filter(|l| l.contains("ec reconstructed interval:"))
+            .map(|l| l.trim_end().to_string())
+            .collect();
+        (read.ok().and_then(Result::ok).map(|n| n.data), lines)
+    }
+
+    const EC_GENERATION: i64 = 1_700_000_000_123_456_789;
+
+    #[derive(Clone, Default)]
+    struct Captured(Arc<std::sync::Mutex<Vec<u8>>>);
+
+    impl std::io::Write for Captured {
+        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+            self.0.lock().unwrap().extend_from_slice(buf);
+            Ok(buf.len())
+        }
+        fn flush(&mut self) -> std::io::Result<()> {
+            Ok(())
+        }
+    }
+
+    impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for Captured {
+        type Writer = Captured;
+        fn make_writer(&'a self) -> Self::Writer {
+            self.clone()
+        }
+    }
+
+    /// What the counter and line mean, case by case. A direct read logs nothing;
+    /// a read that rebuilds shard 0 logs and counts one success; one with 5
+    /// shards left fails and logs one failure; a corrupt interval read directly
+    /// goes to the repair path and isn't marked. `success` is not needle
+    /// validation: rebuilt from corrupt parity, the interval logs success while
+    /// the read fails.
+    #[tokio::test]
+    async fn test_ec_reconstructed_interval() {
+        use crate::metrics::EC_RECONSTRUCTED_INTERVAL_COUNTER as C;
+        let prefix = |vid: u32| {
+            format!(
+                "ec reconstructed interval: volume={vid} generation={EC_GENERATION} needle=7 shard=0 "
+            )
+        };
+        let data = Some(b"ec degraded read".to_vec());
+
+        assert_eq!(
+            read_ec_reconstructed(61, &[], &[]).await,
+            (data.clone(), vec![])
+        );
+
+        let success = C.with_label_values(&["success"]).get();
+        let (read, lines) = read_ec_reconstructed(62, &[0], &[]).await;
+        assert_eq!(read, data);
+        assert!(
+            matches!(&lines[..], [l] if l.ends_with(&(prefix(62) + "result=success error=<nil>"))),
+            "{lines:?}"
+        );
+        assert!(C.with_label_values(&["success"]).get() > success);
+
+        let failure = C.with_label_values(&["failure"]).get();
+        let (read, lines) = read_ec_reconstructed(63, &[0, 6, 7, 8, 9, 10, 11, 12, 13], &[]).await;
+        assert_eq!(read, None);
+        assert!(
+            matches!(&lines[..], [l] if l.contains(&(prefix(63) + "result=failure error=")) && !l.ends_with("error=")),
+            "{lines:?}"
+        );
+        assert!(C.with_label_values(&["failure"]).get() > failure);
+
+        let (read, lines) = read_ec_reconstructed(64, &[], &[0, 10, 11, 12, 13]).await;
+        assert_eq!(
+            (read, lines),
+            (None, vec![]),
+            "an unrepairable corrupt read was marked"
+        );
+
+        let success = C.with_label_values(&["success"]).get();
+        let (read, lines) = read_ec_reconstructed(65, &[0], &[10, 11, 12, 13]).await;
+        assert_eq!(read, None, "a read rebuilt from corrupt parity was served");
+        assert!(
+            matches!(&lines[..], [l] if l.ends_with(&(prefix(65) + "result=success error=<nil>"))),
+            "{lines:?}"
+        );
+        assert!(C.with_label_values(&["success"]).get() > success);
+    }
+
     /// Write `data` as a chunk manifest on a fresh EC volume 2. Chunks stay on volume 1.
     fn put_ec_manifest(
         state: &VolumeServerState,

seaweed-volume/src/server/store_ec.rs

@@ -1356,7 +1356,39 @@ async fn fetch_one_interval(
 
     // Reconstruct: fan-out reads to every other shard at the same
     // (shard_offset, size). Mirrors `recoverOneRemoteEcShardInterval`.
-    recover_one_remote_ec_shard_interval(state, iv, map, &HashSet::from([iv.shard_id])).await
+    let result =
+        recover_one_remote_ec_shard_interval(state, iv, map, &HashSet::from([iv.shard_id])).await;
+    note_ec_reconstructed_interval(iv, &result);
+    result
+}
+
+/// Counts one interval that couldn't be read directly and was reconstructed,
+/// and at debug logs it with the needle and the encode generation, which the
+/// counter can't carry. `success` means the reconstruction produced data; the
+/// needle is validated later. A holder's "deleted" answer with too few shards
+/// rebuilt nothing, so it is a failure, as Go's reconstruct error makes it there.
+fn note_ec_reconstructed_interval(iv: EcInterval, result: &io::Result<(Vec<u8>, bool)>) {
+    let outcome = match result {
+        Ok((buf, _)) if !buf.is_empty() => "success",
+        _ => "failure",
+    };
+    crate::metrics::EC_RECONSTRUCTED_INTERVAL_COUNTER
+        .with_label_values(&[outcome])
+        .inc();
+    // tracing evaluates these arguments only when debug is enabled.
+    tracing::debug!(
+        "ec reconstructed interval: volume={} generation={} needle={} shard={} result={} error={}",
+        iv.vid.0,
+        iv.expected_encode_ts_ns,
+        iv.needle_id,
+        iv.shard_id,
+        outcome,
+        match result {
+            Ok((buf, _)) if !buf.is_empty() => "<nil>".to_string(),
+            Ok(_) => "a holder reported the needle deleted".to_string(),
+            Err(e) => e.to_string(),
+        }
+    );
 }
 
 async fn read_remote_ec_shard_interval(