EC: handle zero-sized shard files uniformly (moves, rebuilds, startup cleanup) (#10753)

* volume_move: treat zero-sized EC shards as absent in move verification

A zero-sized shard file is residue of a failed operation (issue 10730),
not a shard - but VerifyEcShards only checked presence, so a copy that
landed as an empty file passed verification and the source was deleted
behind it. Size zero now reads as absent, with a distinct error naming
the zero-sized shard so the operator can tell a broken copy from a
missing one.

* storage: exclude zero-sized EC shards from rebuilds and clean up stale ones

The reproducer in issue 10730: a zero-sized shard file left by a failed
operation was selected as a Reed-Solomon input and failed the whole
rebuild with an input size mismatch, because input discovery checked
existence, not substance.

- RebuildEcFiles treats a zero-sized shard file as missing and
  regenerates over it in place (the reclassified-corrupt path: temp
  file beside the residue, atomic rename).
- The startup/rescan shard loader, which always skipped zero-sized
  files, now deletes them once they are older than an hour - young
  enough files can be an in-flight copy's just-created file, since the
  same scan runs from LoadNewVolumes while serving.

Regression tests: a rebuild with one emptied shard regenerates it
byte-identical; the loader deletes a stale zero-sized shard and leaves
a fresh one alone.

* storage: age-check each zero-shard cleanup candidate individually

The shard scan merges the data and idx directory listings, so the
age-checked entry and a deletion candidate can be different files
sharing one name - a stale zero-sized file in one directory next to a
fresh same-named file in the other (possibly an in-flight copy's
just-created one) could get the fresh file deleted. Each candidate's
own modification time now decides, both directories are handled in one
pass, and the split-directory case is pinned by a test.

weed/storage/erasure_coding/ec_decoder_test.go

@@ -542,13 +542,13 @@ func TestEcxFileDeletionWithSeparateHandles(t *testing.T) {
 // are journaled to .ecj and tracked in an in-memory set — so the
 // durability chain decode relies on is:
 //
-//   1. DeleteNeedleFromEcx appends the needle id to .ecj and fsyncs it.
-//   2. Runtime reads via FindNeedleFromEcx consult the in-memory set and
-//      return TombstoneFileSize even though the sealed .ecx record on
-//      disk still shows the original size.
-//   3. ec.decode later closes the EcVolume and calls RebuildEcxFile on
-//      the now-quiescent files, which walks .ecj and writes tombstones
-//      into .ecx. CopyFile then reads the rebuilt .ecx.
+//  1. DeleteNeedleFromEcx appends the needle id to .ecj and fsyncs it.
+//  2. Runtime reads via FindNeedleFromEcx consult the in-memory set and
+//     return TombstoneFileSize even though the sealed .ecx record on
+//     disk still shows the original size.
+//  3. ec.decode later closes the EcVolume and calls RebuildEcxFile on
+//     the now-quiescent files, which walks .ecj and writes tombstones
+//     into .ecx. CopyFile then reads the rebuilt .ecx.
 //
 // This test exercises the full chain on a tempdir fixture.
 func TestEcVolumeDeleteDurableToJournal(t *testing.T) {

weed/storage/erasure_coding/ec_encoder.go

@@ -167,21 +167,33 @@ func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize
 	shardPaths := make([]string, ctx.Total()) // non-empty for present shards (also the in-place output for a reclassified-corrupt shard)
 	inputFiles := make([]*os.File, ctx.Total())
 	presentCount := 0
+	var zeroSized []int
 	for shardId := 0; shardId < ctx.Total(); shardId++ {
 		ext := ctx.ToExt(shardId)
 		shardPath := findShardFile(baseFileName, ext, additionalDirs)
-		if shardPath != "" {
-			shardHasData[shardId] = true
-			shardPaths[shardId] = shardPath
-			inputFiles[shardId], err = os.OpenFile(shardPath, os.O_RDONLY, 0)
-			if err != nil {
-				return nil, err
-			}
-			defer inputFiles[shardId].Close()
-			presentCount++
-		} else {
+		if shardPath == "" {
 			generatedShardIds = append(generatedShardIds, uint32(shardId))
+			continue
 		}
+		if fi, statErr := os.Stat(shardPath); statErr == nil && fi.Size() == 0 {
+			// A zero-sized shard file is residue of a failed operation, not a
+			// shard; feeding it to Reed-Solomon fails the whole rebuild with a
+			// size mismatch. Treat it as missing and regenerate over it in
+			// place, like a reclassified-corrupt shard.
+			glog.Warningf("shard %d for %s is zero-sized at %s; excluding from rebuild inputs and regenerating", shardId, baseFileName, shardPath)
+			shardPaths[shardId] = shardPath
+			zeroSized = append(zeroSized, shardId)
+			generatedShardIds = append(generatedShardIds, uint32(shardId))
+			continue
+		}
+		shardHasData[shardId] = true
+		shardPaths[shardId] = shardPath
+		inputFiles[shardId], err = os.OpenFile(shardPath, os.O_RDONLY, 0)
+		if err != nil {
+			return nil, err
+		}
+		defer inputFiles[shardId].Close()
+		presentCount++
 	}
 
 	// Bitrot verify-and-exclude: when a generation-0 checksum sidecar is present
@@ -190,6 +202,9 @@ func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize
 	// silently consuming corrupt bytes. corruptOwned marks shards whose
 	// (corrupt) original file must be replaced in place at its discovered path.
 	corruptOwned := make([]bool, ctx.Total())
+	for _, shardId := range zeroSized {
+		corruptOwned[shardId] = true
+	}
 	prot, status := loadRebuildSidecar(baseFileName, ctx, additionalDirs)
 	switch status {
 	case BitrotInvalid:

weed/storage/erasure_coding/ec_rebuild_safety_test.go

@@ -215,6 +215,51 @@ func TestRebuildEcFiles_HappyPathRebuildsByteIdentical(t *testing.T) {
 	}
 }
 
+// TestRebuildEcFiles_ZeroSizedShardRegeneratedInPlace: a zero-sized shard file
+// (residue of a failed operation, issue 10730) must be excluded from the
+// rebuild inputs and regenerated byte-identical over the empty file, instead
+// of failing the whole rebuild with an input size mismatch.
+func TestRebuildEcFiles_ZeroSizedShardRegeneratedInPlace(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "vol")
+	ctx := NewDefaultECContext("", 0)
+	writeRandomDat(t, base, 7000)
+
+	if _, err := generateEcFiles(base, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx); err != nil {
+		t.Fatalf("generateEcFiles: %v", err)
+	}
+
+	const emptied = 2
+	want, err := os.ReadFile(base + ToExt(emptied))
+	if err != nil {
+		t.Fatalf("read shard %d: %v", emptied, err)
+	}
+	if err := os.Truncate(base+ToExt(emptied), 0); err != nil {
+		t.Fatalf("truncate shard: %v", err)
+	}
+
+	generated, err := RebuildEcFiles(base, ctx, true)
+	if err != nil {
+		t.Fatalf("RebuildEcFiles with a zero-sized shard: %v", err)
+	}
+	found := false
+	for _, id := range generated {
+		if id == emptied {
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("shard %d not reported as regenerated: %v", emptied, generated)
+	}
+	got, err := os.ReadFile(base + ToExt(emptied))
+	if err != nil {
+		t.Fatalf("read regenerated shard: %v", err)
+	}
+	if string(got) != string(want) {
+		t.Errorf("regenerated shard %d differs from original (%d vs %d bytes)", emptied, len(got), len(want))
+	}
+}
+
 // TestRebuildEcFiles_CustomRatioRebuildsByteIdentical: the same for a 9+3 ratio,
 // confirming no 10+4 assumption leaks into the destructive rebuild path.
 func TestRebuildEcFiles_CustomRatioRebuildsByteIdentical(t *testing.T) {