ec: recover EC shards whose .ecx index lives only on a peer server (#10108)

* ec: recover EC shards whose .ecx index lives only on a peer server

A volume server that boots with EC shard files on disk but no .ecx index
on any local disk cannot mount the shards, so the master never learns
about them. ec.rebuild works off master-registered shards, so it sees the
volume as short and gives up even though the shard data is intact.

Add an operator-triggered recovery: VolumeEcShardsMount gains a
recover_missing_index flag that makes the volume server fetch the missing
.ecx (plus .ecj/.vif) from a peer holding it and mount the on-disk shards.
ec.rebuild runs this across the cluster before planning, so orphaned
shards register and the rebuild sees the true shard set.

.ecx is an immutable encode-time index, identical on every holder. .ecj
is a per-holder deletion journal that differs across holders, so the
recovered node adopts the source peer's deletion view, like a balanced or
rebuilt shard does.

* ec: mirror missing-index recovery into the Rust volume server

Port the #10104 recovery to seaweed-volume so the Rust volume server
self-heals the same layout: EC shards on disk with the .ecx index only on
a peer. Adds collect_ec_volumes_missing_index / mount_recovered_ec_shards
to the store, recover_missing_ec_indexes (master LookupEcVolume + peer
CopyFile fetch + mount) to the server, and the recover_missing_index flag
on VolumeEcShardsMount.

.ecx is the immutable encode-time index, identical on every holder. .ecj
is a per-holder deletion journal, so the recovered node adopts the source
peer's deletion view, matching the Go path.

weed/server/volume_grpc_erasure_coding.go

@@ -717,6 +717,14 @@ func (vs *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_ser
 
 	glog.V(0).Infof("VolumeEcShardsMount: %v", req)
 
+	// Fetch a missing .ecx from a peer first so on-disk shards that never had a
+	// local index can be mounted (issue #10104). Driven on demand by ec.rebuild.
+	// volume_id 0 recovers every orphan on this server, including volumes the
+	// master never learned about.
+	if req.RecoverMissingIndex {
+		vs.recoverMissingEcIndexes(req.VolumeId)
+	}
+
 	for _, shardId := range req.ShardIds {
 		err := vs.store.MountEcShards(req.Collection, needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId), req.SourceDiskType)
 

weed/server/volume_grpc_erasure_coding_recover.go

@@ -0,0 +1,168 @@
+package weed_server
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"os"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+)
+
+// ecRecoveryLookupTimeout bounds the master LookupEcVolume call so a slow or
+// unresponsive master cannot hang the synchronous recovery RPC.
+const ecRecoveryLookupTimeout = time.Minute
+
+// recoverMissingEcIndexes fetches the .ecx / .ecj / .vif index files for EC
+// volumes whose shards sit on this server while the index lives only on a peer,
+// then mounts the now-recoverable shards. This self-heals the "shards present,
+// .ecx missing everywhere local" layout (issue #10104) that the per-disk loader
+// and the same-server cross-disk reconcile both leave unmounted, including for
+// volumes already broken before the upgrade.
+//
+// filterVid limits recovery to a single volume id; 0 recovers every orphan on
+// this server (used by ec.rebuild to heal volumes the master never learned
+// about). It is driven on demand by VolumeEcShardsMount (recover_missing_index),
+// so an operator triggers it through ec.rebuild rather than a background loop.
+// Returns the number of volumes whose index was recovered.
+func (vs *VolumeServer) recoverMissingEcIndexes(filterVid uint32) int {
+	orphans := vs.store.CollectEcVolumesMissingIndex()
+	if filterVid != 0 {
+		filtered := orphans[:0]
+		for _, m := range orphans {
+			if uint32(m.VolumeId) == filterVid {
+				filtered = append(filtered, m)
+			}
+		}
+		orphans = filtered
+	}
+	if len(orphans) == 0 {
+		return 0
+	}
+
+	master := vs.getCurrentMaster()
+	if master == "" {
+		glog.Warningf("cannot recover missing EC index without a master connection")
+		return 0
+	}
+	self := pb.NewServerAddress(vs.store.Ip, vs.store.Port, vs.store.GrpcPort)
+
+	recovered := 0
+	for _, m := range orphans {
+		peers, err := vs.lookupEcVolumePeers(master, uint32(m.VolumeId), self)
+		if err != nil {
+			glog.Warningf("ec volume %d: cannot look up peers to recover missing .ecx: %v", m.VolumeId, err)
+			continue
+		}
+		if len(peers) == 0 {
+			glog.Warningf("ec volume %d: shards present locally but .ecx missing and no peer holds it; leaving shards unloaded", m.VolumeId)
+			continue
+		}
+		if vs.fetchEcIndexFromPeers(peers, m) {
+			recovered++
+		}
+	}
+
+	if recovered > 0 {
+		vs.store.MountRecoveredEcShards()
+		glog.V(0).Infof("recovered missing EC index for %d volume(s) from peers and mounted their shards", recovered)
+	}
+	return recovered
+}
+
+// lookupEcVolumePeers asks the master which servers hold shards for vid and
+// returns the unique peer addresses, excluding this server itself.
+func (vs *VolumeServer) lookupEcVolumePeers(master pb.ServerAddress, vid uint32, self pb.ServerAddress) ([]pb.ServerAddress, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), ecRecoveryLookupTimeout)
+	defer cancel()
+
+	var peers []pb.ServerAddress
+	seen := make(map[pb.ServerAddress]bool)
+	err := operation.WithMasterServerClient(ctx, false, master, vs.grpcDialOption, func(client master_pb.SeaweedClient) error {
+		resp, err := client.LookupEcVolume(ctx, &master_pb.LookupEcVolumeRequest{VolumeId: vid})
+		if err != nil {
+			return err
+		}
+		for _, shardIdLocations := range resp.ShardIdLocations {
+			for _, loc := range shardIdLocations.Locations {
+				addr := pb.NewServerAddressFromLocation(loc)
+				if addr.Equals(self) || seen[addr] {
+					continue
+				}
+				seen[addr] = true
+				peers = append(peers, addr)
+			}
+		}
+		return nil
+	})
+	return peers, err
+}
+
+// fetchEcIndexFromPeers tries each peer in turn, copying the .ecx (required) and
+// the .ecj / .vif (best-effort) for the volume onto the local disk recorded in
+// m. The .ecx is an immutable encode-time index, identical on every holder, so
+// any peer's copy serves. The .ecj is a per-holder deletion journal that differs
+// across holders (a delete is journaled on only one node); the recovered node
+// adopts the source peer's deletion view, exactly as a balanced or rebuilt shard
+// does — the EC delete model already tolerates that divergence. The first peer
+// that yields a non-empty .ecx wins; a peer with no index or a 0-byte stub is
+// skipped (the orphan shards are non-empty, so a 0-byte index cannot be theirs).
+// A failed copy removes its partial file so a later attempt is not blocked by a
+// stub.
+func (vs *VolumeServer) fetchEcIndexFromPeers(peers []pb.ServerAddress, m storage.EcVolumeMissingIndex) bool {
+	idxBaseFileName := storage.VolumeFileName(m.IdxDir, m.Collection, int(m.VolumeId))
+	dataBaseFileName := storage.VolumeFileName(m.DataDir, m.Collection, int(m.VolumeId))
+	ecxPath := idxBaseFileName + ".ecx"
+	ecjPath := idxBaseFileName + ".ecj"
+
+	removePartial := func(path string) {
+		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+			glog.Warningf("ec volume %d: remove partial %s: %v", m.VolumeId, path, err)
+		}
+	}
+
+	for _, peer := range peers {
+		err := operation.WithVolumeServerClient(true, peer, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+			// .ecx is mandatory; a peer without it errors and we move on.
+			if _, err := vs.doCopyFile(client, true, m.Collection, uint32(m.VolumeId), math.MaxUint32, math.MaxInt64, idxBaseFileName, ".ecx", false, false, nil); err != nil {
+				removePartial(ecxPath)
+				return err
+			}
+			info, statErr := os.Stat(ecxPath)
+			if statErr != nil {
+				removePartial(ecxPath)
+				return fmt.Errorf("stat copied .ecx %s: %w", ecxPath, statErr)
+			}
+			if info.IsDir() || info.Size() == 0 {
+				removePartial(ecxPath)
+				return fmt.Errorf("peer %s served an unusable .ecx (size %d)", peer, info.Size())
+			}
+			// .ecj is the source peer's deletion journal; .vif carries EC params
+			// and EncodeTsNs. Both are best-effort: a missing .ecj is recreated at
+			// mount and a missing .vif falls back to default EC parameters. A failed
+			// .ecj copy is an in-place append, so drop the partial file; the .vif
+			// copy stages and renames, leaving nothing to clean up.
+			if _, err := vs.doCopyFile(client, true, m.Collection, uint32(m.VolumeId), math.MaxUint32, math.MaxInt64, idxBaseFileName, ".ecj", true, true, nil); err != nil {
+				glog.Warningf("ec volume %d: copy .ecj from %s: %v", m.VolumeId, peer, err)
+				removePartial(ecjPath)
+			}
+			if _, err := vs.doCopyFile(client, true, m.Collection, uint32(m.VolumeId), math.MaxUint32, math.MaxInt64, dataBaseFileName, ".vif", false, true, nil); err != nil {
+				glog.Warningf("ec volume %d: copy .vif from %s: %v", m.VolumeId, peer, err)
+			}
+			return nil
+		})
+		if err != nil {
+			glog.V(1).Infof("ec volume %d: fetch missing .ecx from %s failed: %v", m.VolumeId, peer, err)
+			continue
+		}
+		glog.V(0).Infof("ec volume %d: fetched missing .ecx from %s into %s", m.VolumeId, peer, m.IdxDir)
+		return true
+	}
+	return false
+}

weed/server/volume_grpc_erasure_coding_recover_test.go

@@ -0,0 +1,285 @@
+package weed_server
+
+import (
+	"context"
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// fakeMaster serves only LookupEcVolume, returning the configured holders for
+// every shard of one volume. Used to drive the receiver's peer discovery.
+type fakeMaster struct {
+	master_pb.UnimplementedSeaweedServer
+	volumeId  uint32
+	locations []*master_pb.Location
+}
+
+func (m *fakeMaster) LookupEcVolume(_ context.Context, req *master_pb.LookupEcVolumeRequest) (*master_pb.LookupEcVolumeResponse, error) {
+	resp := &master_pb.LookupEcVolumeResponse{VolumeId: req.VolumeId}
+	if req.VolumeId != m.volumeId {
+		return resp, nil
+	}
+	// Recovery only needs one peer holding the index; report a couple of shards
+	// pointing at the holder, independent of the EC ratio.
+	for shardId := 0; shardId < 2; shardId++ {
+		resp.ShardIdLocations = append(resp.ShardIdLocations, &master_pb.LookupEcVolumeResponse_EcShardIdLocation{
+			ShardId:   uint32(shardId),
+			Locations: m.locations,
+		})
+	}
+	return resp, nil
+}
+
+// serveGrpc registers register(s) on a fresh localhost listener and returns its
+// grpc port, stopping the server on test cleanup.
+func serveGrpc(t *testing.T, register func(*grpc.Server)) int {
+	t.Helper()
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	srv := grpc.NewServer()
+	register(srv)
+	go srv.Serve(lis)
+	t.Cleanup(srv.Stop)
+	return lis.Addr().(*net.TCPAddr).Port
+}
+
+func newRecoverTestStore(t *testing.T, dir string) *storage.Store {
+	t.Helper()
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatalf("mkdir %s: %v", dir, err)
+	}
+	store := storage.NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id",
+		[]string{dir}, []int32{100}, []util.MinFreeSpace{{}}, "",
+		storage.NeedleMapInMemory, []types.DiskType{types.HardDriveType}, nil, 3, stats.DefaultDiskIOProbeConfig())
+	done := make(chan struct{})
+	go func() {
+		for {
+			select {
+			case <-store.NewEcShardsChan:
+			case <-store.NewVolumesChan:
+			case <-store.DeletedVolumesChan:
+			case <-store.DeletedEcShardsChan:
+			case <-store.StateUpdateChan:
+			case <-done:
+				return
+			}
+		}
+	}()
+	t.Cleanup(func() {
+		store.Close()
+		close(done)
+	})
+	return store
+}
+
+// TestFetchEcIndexFromPeers_CopiesIndexOverGrpc stands up a source volume server
+// that holds the .ecx/.ecj/.vif for a volume and verifies the receiver pulls
+// them over a real CopyFile gRPC stream into its own disk, the way #10104
+// recovery does when the index lives only on a peer.
+func TestFetchEcIndexFromPeers_CopiesIndexOverGrpc(t *testing.T) {
+	const collection = "video-recordings"
+	vid := needle.VolumeId(6190)
+
+	// Source server: has the index files on disk.
+	srcDir := filepath.Join(t.TempDir(), "src")
+	srcStore := newRecoverTestStore(t, srcDir)
+	srcBase := erasure_coding.EcShardFileName(collection, srcDir, int(vid))
+	ecxBytes := make([]byte, types.NeedleMapEntrySize*3)
+	for i := range ecxBytes {
+		ecxBytes[i] = byte(i)
+	}
+	if err := os.WriteFile(srcBase+".ecx", ecxBytes, 0o644); err != nil {
+		t.Fatalf("write source .ecx: %v", err)
+	}
+	if err := os.WriteFile(srcBase+".ecj", []byte("journal"), 0o644); err != nil {
+		t.Fatalf("write source .ecj: %v", err)
+	}
+	if err := os.WriteFile(srcBase+".vif", []byte("volinfo"), 0o644); err != nil {
+		t.Fatalf("write source .vif: %v", err)
+	}
+
+	// Serve the source over a real TCP gRPC listener.
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	grpcServer := grpc.NewServer()
+	volume_server_pb.RegisterVolumeServerServer(grpcServer, &VolumeServer{store: srcStore})
+	go grpcServer.Serve(lis)
+	t.Cleanup(grpcServer.Stop)
+
+	grpcPort := lis.Addr().(*net.TCPAddr).Port
+	peer := pb.NewServerAddress("127.0.0.1", grpcPort-10000, grpcPort)
+
+	// Receiver server: empty disk, ready to receive the index.
+	dstDir := filepath.Join(t.TempDir(), "dst")
+	dstStore := newRecoverTestStore(t, dstDir)
+	receiver := &VolumeServer{
+		store:          dstStore,
+		grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+	}
+
+	m := storage.EcVolumeMissingIndex{
+		Collection: collection,
+		VolumeId:   vid,
+		IdxDir:     dstDir,
+		DataDir:    dstDir,
+	}
+	if !receiver.fetchEcIndexFromPeers([]pb.ServerAddress{peer}, m) {
+		t.Fatalf("fetchEcIndexFromPeers returned false; expected a successful copy")
+	}
+
+	dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid))
+	got, err := os.ReadFile(dstBase + ".ecx")
+	if err != nil {
+		t.Fatalf("read copied .ecx: %v", err)
+	}
+	if len(got) != len(ecxBytes) {
+		t.Errorf("copied .ecx size = %d, want %d", len(got), len(ecxBytes))
+	}
+	if _, err := os.Stat(dstBase + ".ecj"); err != nil {
+		t.Errorf("copied .ecj missing: %v", err)
+	}
+	if _, err := os.Stat(dstBase + ".vif"); err != nil {
+		t.Errorf("copied .vif missing: %v", err)
+	}
+}
+
+// TestFetchEcIndexFromPeers_SkipsPeerWithoutIndex verifies the receiver moves on
+// to the next peer when the first has no .ecx, and reports failure when no peer
+// can serve a usable index.
+func TestFetchEcIndexFromPeers_SkipsPeerWithoutIndex(t *testing.T) {
+	const collection = "video-recordings"
+	vid := needle.VolumeId(6191)
+
+	// Source server with NO index files for this volume.
+	srcDir := filepath.Join(t.TempDir(), "src")
+	srcStore := newRecoverTestStore(t, srcDir)
+
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	grpcServer := grpc.NewServer()
+	volume_server_pb.RegisterVolumeServerServer(grpcServer, &VolumeServer{store: srcStore})
+	go grpcServer.Serve(lis)
+	t.Cleanup(grpcServer.Stop)
+
+	grpcPort := lis.Addr().(*net.TCPAddr).Port
+	peer := pb.NewServerAddress("127.0.0.1", grpcPort-10000, grpcPort)
+
+	dstDir := filepath.Join(t.TempDir(), "dst")
+	dstStore := newRecoverTestStore(t, dstDir)
+	receiver := &VolumeServer{
+		store:          dstStore,
+		grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+	}
+
+	m := storage.EcVolumeMissingIndex{Collection: collection, VolumeId: vid, IdxDir: dstDir, DataDir: dstDir}
+	if receiver.fetchEcIndexFromPeers([]pb.ServerAddress{peer}, m) {
+		t.Fatalf("fetchEcIndexFromPeers should fail when no peer has the index")
+	}
+
+	// No stub .ecx must be left behind.
+	dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid))
+	if _, err := os.Stat(dstBase + ".ecx"); !os.IsNotExist(err) {
+		t.Errorf("a .ecx stub was left behind; stat err = %v", err)
+	}
+}
+
+// TestVolumeEcShardsMount_RecoverMissingIndex drives the full on-demand path:
+// VolumeEcShardsMount with recover_missing_index (volume_id 0 = recover every
+// orphan on this server, as ec.rebuild broadcasts it) looks up holders via a
+// (fake) master, fetches the missing .ecx/.ecj/.vif from the holding peer over
+// gRPC, and mounts the previously-orphaned on-disk shards (issue #10104).
+func TestVolumeEcShardsMount_RecoverMissingIndex(t *testing.T) {
+	const collection = "video-recordings"
+	vid := needle.VolumeId(6190)
+
+	// Holder peer: serves the index files for the volume.
+	srcDir := filepath.Join(t.TempDir(), "src")
+	srcStore := newRecoverTestStore(t, srcDir)
+	srcBase := erasure_coding.EcShardFileName(collection, srcDir, int(vid))
+	if err := os.WriteFile(srcBase+".ecx", make([]byte, types.NeedleMapEntrySize*4), 0o644); err != nil {
+		t.Fatalf("write source .ecx: %v", err)
+	}
+	if err := os.WriteFile(srcBase+".ecj", nil, 0o644); err != nil {
+		t.Fatalf("write source .ecj: %v", err)
+	}
+	if err := os.WriteFile(srcBase+".vif", []byte("volinfo"), 0o644); err != nil {
+		t.Fatalf("write source .vif: %v", err)
+	}
+	srcGrpcPort := serveGrpc(t, func(s *grpc.Server) {
+		volume_server_pb.RegisterVolumeServerServer(s, &VolumeServer{store: srcStore})
+	})
+
+	// Fake master points every shard at the holder peer.
+	masterGrpcPort := serveGrpc(t, func(s *grpc.Server) {
+		master_pb.RegisterSeaweedServer(s, &fakeMaster{
+			volumeId:  uint32(vid),
+			locations: []*master_pb.Location{{Url: "127.0.0.1:1", GrpcPort: uint32(srcGrpcPort)}},
+		})
+	})
+
+	// Receiver: holds orphan shard files on disk, no .ecx anywhere.
+	dstDir := filepath.Join(t.TempDir(), "dst")
+	dstStore := newRecoverTestStore(t, dstDir)
+	const shardSize = 1 << 20
+	for _, sid := range []erasure_coding.ShardId{0, 5} {
+		base := erasure_coding.EcShardFileName(collection, dstDir, int(vid))
+		f, err := os.Create(base + erasure_coding.ToExt(int(sid)))
+		if err != nil {
+			t.Fatalf("create shard %d: %v", sid, err)
+		}
+		if err := f.Truncate(shardSize); err != nil {
+			f.Close()
+			t.Fatalf("truncate shard %d: %v", sid, err)
+		}
+		f.Close()
+	}
+
+	receiver := &VolumeServer{
+		store:          dstStore,
+		grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+	}
+	receiver.setCurrentMaster(pb.NewServerAddress("127.0.0.1", masterGrpcPort-10000, masterGrpcPort))
+
+	if _, found := dstStore.FindEcVolume(vid); found {
+		t.Fatalf("EC volume %d unexpectedly mounted before recovery", vid)
+	}
+
+	// volume_id 0: the receiver discovers the orphan (6190) on disk itself, even
+	// though the request names no volume and the master never registered it here.
+	if _, err := receiver.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
+		RecoverMissingIndex: true,
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsMount with recover_missing_index: %v", err)
+	}
+
+	ev, found := dstStore.FindEcVolume(vid)
+	if !found {
+		t.Fatalf("EC volume %d not mounted after recovery", vid)
+	}
+	for _, sid := range []erasure_coding.ShardId{0, 5} {
+		if _, ok := ev.FindEcVolumeShard(sid); !ok {
+			t.Errorf("shard %d.%d not registered after recovery", vid, sid)
+		}
+	}
+}