ec: recover EC shards whose .ecx index lives only on a peer server (#10108)

* ec: recover EC shards whose .ecx index lives only on a peer server

A volume server that boots with EC shard files on disk but no .ecx index
on any local disk cannot mount the shards, so the master never learns
about them. ec.rebuild works off master-registered shards, so it sees the
volume as short and gives up even though the shard data is intact.

Add an operator-triggered recovery: VolumeEcShardsMount gains a
recover_missing_index flag that makes the volume server fetch the missing
.ecx (plus .ecj/.vif) from a peer holding it and mount the on-disk shards.
ec.rebuild runs this across the cluster before planning, so orphaned
shards register and the rebuild sees the true shard set.

.ecx is an immutable encode-time index, identical on every holder. .ecj
is a per-holder deletion journal that differs across holders, so the
recovered node adopts the source peer's deletion view, like a balanced or
rebuilt shard does.

* ec: mirror missing-index recovery into the Rust volume server

Port the #10104 recovery to seaweed-volume so the Rust volume server
self-heals the same layout: EC shards on disk with the .ecx index only on
a peer. Adds collect_ec_volumes_missing_index / mount_recovered_ec_shards
to the store, recover_missing_ec_indexes (master LookupEcVolume + peer
CopyFile fetch + mount) to the server, and the recover_missing_index flag
on VolumeEcShardsMount.

.ecx is the immutable encode-time index, identical on every holder. .ecj
is a per-holder deletion journal, so the recovered node adopts the source
peer's deletion view, matching the Go path.

weed/shell/command_ec_rebuild.go

@@ -47,6 +47,11 @@ func (c *commandEcRebuild) Help() string {
 
 	ec.rebuild [-c EACH_COLLECTION|<collection_name>] [-apply] [-maxParallelization N] [-diskType=<disk_type>]
 
+	Before rebuilding, asks volume servers to recover any shards left unmounted by
+	a missing .ecx index (the index resides only on a peer server). Such shards are
+	invisible to the master, so recovering them first avoids regenerating data that
+	is actually present (issue #10104).
+
 	Options:
 	  -collection: specify a collection name, or "EACH_COLLECTION" to process all collections
 	  -apply: actually perform the rebuild operations (default is dry-run mode)
@@ -149,6 +154,11 @@ func (c *commandEcRebuild) Do(args []string, commandEnv *CommandEnv, writer io.W
 		ewg: NewErrorWaitGroup(*maxParallelization),
 	}
 
+	// Recover shards left unmounted by a missing .ecx index before planning: such
+	// shards never register with the master, so the rebuild below would treat the
+	// volume as short or unrepairable even though its data is intact (issue #10104).
+	erb.recoverMissingIndexes()
+
 	fmt.Printf("rebuildEcVolumes for %d collection(s)\n", len(collections))
 	for _, c := range collections {
 		erb.rebuildEcVolumes(c)
@@ -288,6 +298,61 @@ func (erb *ecRebuilder) rebuildEcVolumes(collection string) {
 	}
 }
 
+// recoverMissingIndexes asks every ec node to fetch a missing .ecx index from a
+// peer and mount the on-disk shards it could not load on its own. Shards
+// orphaned this way (index only on another server) are absent from the master
+// topology, so without this pass ec.rebuild would regenerate or give up on
+// shards whose data is actually present — and a volume whose every holder lacks
+// the index would not appear in the topology at all. Each node therefore
+// recovers all of its on-disk orphans (volume_id 0); an explicit -volumeIds
+// list narrows that to the requested volumes. On apply it refreshes the topology
+// so the rebuild planning sees the recovered shards (issue #10104).
+func (erb *ecRebuilder) recoverMissingIndexes() {
+	erb.ecNodesMu.Lock()
+	nodes := append([]*EcNode(nil), erb.ecNodes...)
+	erb.ecNodesMu.Unlock()
+	if len(nodes) == 0 {
+		return
+	}
+
+	// volume_id 0 means "recover every orphan on the node"; a -volumeIds list
+	// narrows recovery to those ids (each scanned across collections server-side).
+	vids := erb.volumeIds
+	if len(vids) == 0 {
+		vids = []needle.VolumeId{0}
+	}
+
+	if !erb.applyChanges {
+		erb.write("would ask %d ec node(s) to recover EC shards left unmounted by a missing .ecx index\n", len(nodes))
+		return
+	}
+
+	for _, node := range nodes {
+		for _, vid := range vids {
+			err := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(node.info), erb.commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+				_, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
+					VolumeId:            uint32(vid),
+					RecoverMissingIndex: true,
+				})
+				return mountErr
+			})
+			if err != nil {
+				erb.write("%s recover missing index (volume %d): %v\n", node.info.Id, vid, err)
+			}
+		}
+	}
+
+	// Refresh topology so the rebuild planning sees shards the recovery registered.
+	refreshed, _, err := collectEcNodes(erb.commandEnv, erb.diskType)
+	if err != nil {
+		erb.write("failed to refresh ec nodes after index recovery: %v\n", err)
+		return
+	}
+	erb.ecNodesMu.Lock()
+	erb.ecNodes = refreshed
+	erb.ecNodesMu.Unlock()
+}
+
 func (erb *ecRebuilder) rebuildOneEcVolume(collection string, volumeId needle.VolumeId, locations EcShardLocations, rebuilder *EcNode) error {
 	if !erb.isLocked() {
 		return fmt.Errorf("lock is lost")