volume: count EC intervals served by reconstruction (#12392)

* volume: count EC reads that fall back to reconstruction

When a needle read can't read an EC shard interval directly (the shard isn't
mounted here and no holder answers), both volume servers rebuild it from the
other shards and the read still succeeds, with nothing to show for it.

Count those fallbacks in SeaweedFS_volumeServer_ec_degraded_read_total{result}
(success, failure), one per interval, at the one place each server falls back:
Go readOneEcShardInterval, Rust fetch_one_interval. Corruption repair and scrub
aren't counted. A line binding each one to the needle and the volume's encode
generation is logged at V(3) in Go and debug in Rust.

* volume: name the EC counter for what it counts

Reed-Solomon reconstruction can't tell corrupt inputs from good ones, and the
needle is validated later, so "success" only ever meant the reconstruction
produced data. Rename the counter to
SeaweedFS_volumeServer_ec_reconstructed_interval_total and the log line to
"ec reconstructed interval", say so in the help text, and pin the behaviour
with a test: an interval rebuilt from corrupt parity counts success while the
read itself fails.

* storage: restore stderr when the log-line test stops early

A fixture failure in readEcNeedle ends the test through t.Fatalf, which
skipped restoring os.Stderr and closing the pipe, so later tests wrote
into a pipe nobody read.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

weed/stats/metrics.go

@@ -668,6 +668,18 @@ var (
 			Help:      "Counter of EC shard rebuild operations by result.",
 		}, []string{"result"})
 
+	// VolumeServerECReconstructedIntervalCounter counts reads of an EC shard
+	// interval that could not be read directly and were served by reconstructing
+	// it from the other shards. success means the reconstruction produced data,
+	// not that the needle then passed validation.
+	VolumeServerECReconstructedIntervalCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: subsystemVolumeServer,
+			Name:      "ec_reconstructed_interval_total",
+			Help:      "Reads of an EC interval served by reconstructing from other shards; success means reconstruction produced data, not that the needle passed validation.",
+		}, []string{"result"})
+
 	S3RequestCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: Namespace,
@@ -1077,6 +1089,7 @@ func init() {
 	Gather.MustRegister(VolumeServerReplicationFailures)
 	Gather.MustRegister(VolumeServerECRebuildHistogram)
 	Gather.MustRegister(VolumeServerECRebuildCounter)
+	Gather.MustRegister(VolumeServerECReconstructedIntervalCounter)
 	Gather.MustRegister(MasterUnderReplicatedVolumes)
 	Gather.MustRegister(MasterVolumeCreationCounter)
 

weed/storage/store_ec.go

@@ -682,6 +682,7 @@ func (s *Store) readOneEcShardInterval(needleId types.NeedleId, ecVolume *erasur
 
 	// try reading by recovering from other shards
 	_, is_deleted, err = s.recoverOneRemoteEcShardInterval(needleId, ecVolume, shardId, data, actualOffset)
+	noteEcReconstructedInterval(ecVolume, needleId, shardId, err)
 	if err == nil {
 		return
 	}
@@ -1264,6 +1265,20 @@ func (s *Store) recoverEcShardInterval(needleId types.NeedleId, ecVolume *erasur
 	return len(buf), is_deleted, nil
 }
 
+// noteEcReconstructedInterval counts one shard interval that could not be read
+// directly and was reconstructed from the other shards, and at V(3) logs it with
+// the needle and the volume's encode generation, which the counter can't carry.
+// success means the reconstruction produced data; the needle is validated later.
+func noteEcReconstructedInterval(ecVolume *erasure_coding.EcVolume, needleId types.NeedleId, shardId erasure_coding.ShardId, err error) {
+	result := "success"
+	if err != nil {
+		result = "failure"
+	}
+	stats.VolumeServerECReconstructedIntervalCounter.WithLabelValues(result).Inc()
+	glog.V(3).Infof("ec reconstructed interval: volume=%d generation=%d needle=%s shard=%d result=%s error=%v",
+		ecVolume.VolumeId, ecVolume.EncodeTsNs, needleId, shardId, result, err)
+}
+
 func (s *Store) EcVolumes() (ecVolumes []*erasure_coding.EcVolume) {
 	for _, location := range s.Locations {
 		location.ecVolumesLock.RLock()

weed/storage/store_ec_reconstructed_interval_test.go

@@ -0,0 +1,133 @@
+package storage
+
+import (
+	"io"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
+)
+
+type shards = []erasure_coding.ShardId
+
+// reconstructedIntervals reads ec_reconstructed_interval_total; this package
+// has no parallel tests, so a test's before/after difference is its own.
+func reconstructedIntervals() (success, failure float64) {
+	c := stats.VolumeServerECReconstructedIntervalCounter
+	return testutil.ToFloat64(c.WithLabelValues("success")), testutil.ToFloat64(c.WithLabelValues("failure"))
+}
+
+// readEcNeedle writes a one-needle EC volume, overwrites the needle's range on
+// the corrupt shards, mounts every shard but missing (with no location for
+// them, so the read reconstructs), reads the needle, and returns how far each
+// counter moved.
+func readEcNeedle(t *testing.T, vid needle.VolumeId, missing, corrupt shards) (n *needle.Needle, err error, dSuccess, dFailure float64) {
+	t.Helper()
+	store := newTestStore(t, 1)
+	_, n = writeEcVolumeFiles(t, store.Locations[0].Directory, vid)
+	skip := map[erasure_coding.ShardId]bool{}
+	for _, sid := range missing {
+		skip[sid] = true
+	}
+	for sid := erasure_coding.ShardId(0); sid < erasure_coding.TotalShardsCount; sid++ {
+		if skip[sid] {
+			continue
+		}
+		if err := store.MountEcShards("", vid, sid, ""); err != nil {
+			t.Fatalf("mount shard %d: %v", sid, err)
+		}
+	}
+	ecVolume, _ := store.Locations[0].FindEcVolume(vid)
+	ecVolume.EncodeTsNs = 1700000000123456789
+	_, _, intervals, err := ecVolume.LocateEcShardNeedle(n.Id, ecVolume.Version)
+	if err != nil {
+		t.Fatalf("locate needle: %v", err)
+	}
+	_, off := ecVolume.IntervalToShardIdAndOffset(intervals[0])
+	for _, sid := range corrupt {
+		corruptEcShardRange(t, ecVolume, sid, off, int64(intervals[0].Size))
+	}
+	seedShardLocations(ecVolume, pb.NewServerAddressWithGrpcPort("127.0.0.1:1", 1))
+	ecVolume.ShardLocationsLock.Lock()
+	for sid := range skip {
+		delete(ecVolume.ShardLocations, sid)
+	}
+	ecVolume.ShardLocationsLock.Unlock()
+
+	s0, f0 := reconstructedIntervals()
+	got := &needle.Needle{Id: n.Id}
+	if _, err = store.ReadEcShardNeedle(vid, got, nil); err == nil && string(got.Data) != string(n.Data) {
+		t.Fatal("read returned different bytes")
+	}
+	s1, f1 := reconstructedIntervals()
+	return n, err, s1 - s0, f1 - f0
+}
+
+// What the counter means, case by case. A direct read isn't counted, nor is a
+// corrupt interval read directly (that goes to the repair path). A missing
+// shard counts one success when the reconstruction produced data and one
+// failure when it couldn't (5 shards left). success is not needle validation:
+// rebuilt from corrupt parity, the interval counts success while the read fails.
+func TestEcReconstructedIntervalCounter(t *testing.T) {
+	parity := shards{10, 11, 12, 13}
+	for _, c := range []struct {
+		vid              needle.VolumeId
+		missing, corrupt shards
+		fails            bool
+		s, f             float64
+	}{
+		{17, nil, nil, false, 0, 0},
+		{18, shards{0}, nil, false, 1, 0},
+		{19, shards{0, 6, 7, 8, 9, 10, 11, 12, 13}, nil, true, 0, 1},
+		{20, nil, append(shards{0}, parity...), true, 0, 0},
+		{21, shards{0}, parity, true, 1, 0},
+	} {
+		if _, err, s, f := readEcNeedle(t, c.vid, c.missing, c.corrupt); (err != nil) != c.fails || s != c.s || f != c.f {
+			t.Errorf("missing %v, corrupt %v: err %v, success +%v, failure +%v; want failed=%v, +%v, +%v", c.missing, c.corrupt, err, s, f, c.fails, c.s, c.f)
+		}
+	}
+}
+
+// The line binds a reconstruction to the volume's generation and the needle,
+// at V(3) and not below.
+func TestEcReconstructedIntervalLogLine(t *testing.T) {
+	flag.Set("logtostderr", "true")
+	t.Cleanup(func() { flag.Set("vmodule", ""); flag.Set("logtostderr", "false") })
+	markers := func(vmodule string, vid needle.VolumeId, missing shards) (n *needle.Needle, lines []string) {
+		flag.Set("vmodule", vmodule)
+		r, w, _ := os.Pipe()
+		stderr, out := os.Stderr, make(chan []byte, 1)
+		restore := sync.OnceFunc(func() { os.Stderr = stderr; w.Close() })
+		t.Cleanup(restore)
+		go func() { b, _ := io.ReadAll(r); r.Close(); out <- b }()
+		os.Stderr = w
+		n, _, _, _ = readEcNeedle(t, vid, missing, nil)
+		restore()
+		for _, line := range strings.Split(string(<-out), "\n") {
+			if strings.Contains(line, "ec reconstructed interval:") {
+				lines = append(lines, line)
+			}
+		}
+		return n, lines
+	}
+	prefix := func(vid int, n *needle.Needle) string {
+		return "ec reconstructed interval: volume=" + needle.VolumeId(vid).String() + " generation=1700000000123456789 needle=" + n.Id.String() + " shard=0 "
+	}
+	if n, lines := markers("store_ec=3", 22, shards{0}); len(lines) != 1 || !strings.HasSuffix(lines[0], prefix(22, n)+"result=success error=<nil>") {
+		t.Errorf("success line %q", lines)
+	}
+	if n, lines := markers("store_ec=3", 23, shards{0, 6, 7, 8, 9, 10, 11, 12, 13}); len(lines) != 1 ||
+		!strings.Contains(lines[0], prefix(23, n)+"result=failure error=") || strings.HasSuffix(lines[0], "error=<nil>") {
+		t.Errorf("failure line %q", lines)
+	}
+	if _, lines := markers("store_ec=2", 24, shards{0}); len(lines) != 0 {
+		t.Errorf("at V(2) the line was logged: %q", lines)
+	}
+}