seaweedfs/weed/server/volume_grpc_erasure_coding_recover.go

package weed_server

import (
	"context"
	"fmt"
	"math"
	"os"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/operation"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"github.com/seaweedfs/seaweedfs/weed/storage"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

// ecRecoveryLookupTimeout bounds the master LookupEcVolume call so a slow or
// unresponsive master cannot hang the synchronous recovery RPC.
const ecRecoveryLookupTimeout = time.Minute

// recoverMissingEcIndexes fetches the .ecx / .ecj / .vif index files for EC
// volumes whose shards sit on this server while the index lives only on a peer,
// then mounts the now-recoverable shards. This self-heals the "shards present,
// .ecx missing everywhere local" layout (issue #10104) that the per-disk loader
// and the same-server cross-disk reconcile both leave unmounted, including for
// volumes already broken before the upgrade.
//
// filterVid limits recovery to a single volume id; 0 recovers every orphan on
// this server (used by ec.rebuild to heal volumes the master never learned
// about). It is driven on demand by VolumeEcShardsMount (recover_missing_index),
// so an operator triggers it through ec.rebuild rather than a background loop.
// Returns the number of volumes whose index was recovered.
func (vs *VolumeServer) recoverMissingEcIndexes(filterVid uint32) int {
	orphans := vs.store.CollectEcVolumesMissingIndex()
	if filterVid != 0 {
		filtered := orphans[:0]
		for _, m := range orphans {
			if uint32(m.VolumeId) == filterVid {
				filtered = append(filtered, m)
			}
		}
		orphans = filtered
	}
	if len(orphans) == 0 {
		return 0
	}

	master := vs.getCurrentMaster()
	if master == "" {
		glog.Warningf("cannot recover missing EC index without a master connection")
		return 0
	}
	self := pb.NewServerAddress(vs.store.Ip, vs.store.Port, vs.store.GrpcPort)

	recovered := 0
	for _, m := range orphans {
		peers, err := vs.lookupEcVolumePeers(master, uint32(m.VolumeId), self)
		if err != nil {
			glog.Warningf("ec volume %d: cannot look up peers to recover missing .ecx: %v", m.VolumeId, err)
			continue
		}
		if len(peers) == 0 {
			glog.Warningf("ec volume %d: shards present locally but .ecx missing and no peer holds it; leaving shards unloaded", m.VolumeId)
			continue
		}
		if vs.fetchEcIndexFromPeers(peers, m) {
			recovered++
		}
	}

	if recovered > 0 {
		vs.store.MountRecoveredEcShards()
		glog.V(0).Infof("recovered missing EC index for %d volume(s) from peers and mounted their shards", recovered)
	}
	return recovered
}

// lookupEcVolumePeers asks the master which servers hold shards for vid and
// returns the unique peer addresses, excluding this server itself.
func (vs *VolumeServer) lookupEcVolumePeers(master pb.ServerAddress, vid uint32, self pb.ServerAddress) ([]pb.ServerAddress, error) {
	ctx, cancel := context.WithTimeout(context.Background(), ecRecoveryLookupTimeout)
	defer cancel()

	var peers []pb.ServerAddress
	seen := make(map[pb.ServerAddress]bool)
	err := operation.WithMasterServerClient(ctx, false, master, vs.grpcDialOption, func(client master_pb.SeaweedClient) error {
		resp, err := client.LookupEcVolume(ctx, &master_pb.LookupEcVolumeRequest{VolumeId: vid})
		if err != nil {
			return err
		}
		for _, shardIdLocations := range resp.ShardIdLocations {
			for _, loc := range shardIdLocations.Locations {
				addr := pb.NewServerAddressFromLocation(loc)
				if addr.Equals(self) || seen[addr] {
					continue
				}
				seen[addr] = true
				peers = append(peers, addr)
			}
		}
		return nil
	})
	return peers, err
}

// fetchEcIndexFromPeers tries each peer in turn, copying the .ecx (required) and
// the .ecj / .vif (best-effort) for the volume onto the local disk recorded in
// m. The .ecx is an immutable encode-time index, identical on every holder, so
// any peer's copy serves. The .ecj is a per-holder deletion journal that differs
// across holders (a delete is journaled on only one node); the recovered node
// adopts the source peer's deletion view, exactly as a balanced or rebuilt shard
// does — the EC delete model already tolerates that divergence. The first peer
// that yields a non-empty .ecx wins; a peer with no index or a 0-byte stub is
// skipped (the orphan shards are non-empty, so a 0-byte index cannot be theirs).
// A failed copy removes its partial file so a later attempt is not blocked by a
// stub.
func (vs *VolumeServer) fetchEcIndexFromPeers(peers []pb.ServerAddress, m storage.EcVolumeMissingIndex) bool {
	idxBaseFileName := storage.VolumeFileName(m.IdxDir, m.Collection, int(m.VolumeId))
	dataBaseFileName := storage.VolumeFileName(m.DataDir, m.Collection, int(m.VolumeId))
	ecxPath := idxBaseFileName + ".ecx"

	removePartial := func(path string) {
		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
			glog.Warningf("ec volume %d: remove partial %s: %v", m.VolumeId, path, err)
		}
	}

	for _, peer := range peers {
		err := operation.WithVolumeServerClient(true, peer, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
			// .ecx is mandatory; a peer without it errors and we move on.
			if _, err := vs.doCopyFile(client, true, m.Collection, uint32(m.VolumeId), math.MaxUint32, math.MaxInt64, idxBaseFileName, ".ecx", false, false, nil); err != nil {
				removePartial(ecxPath)
				return err
			}
			info, statErr := os.Stat(ecxPath)
			if statErr != nil {
				removePartial(ecxPath)
				return fmt.Errorf("stat copied .ecx %s: %w", ecxPath, statErr)
			}
			if info.IsDir() || info.Size() == 0 {
				removePartial(ecxPath)
				return fmt.Errorf("peer %s served an unusable .ecx (size %d)", peer, info.Size())
			}
			// .ecj is the source peer's deletion journal; .vif carries EC params
			// and EncodeTsNs. Both are best-effort: a missing .ecj is recreated at
			// mount and a missing .vif falls back to default EC parameters. The
			// journal is a *set*: merge it as a union so a bounced volume
			// cannot double it. The merge only appends whole records, and the
			// .vif copy stages and renames, so a failure leaves nothing to clean.
			if err := vs.copyEcjAndMerge(client, m.Collection, uint32(m.VolumeId), m.DataDir, idxBaseFileName, util.NewWriteThrottler(vs.maintenanceBytePerSecond)); err != nil {
				glog.Warningf("ec volume %d: copy .ecj from %s: %v", m.VolumeId, peer, err)
			}
			if _, err := vs.doCopyFile(client, true, m.Collection, uint32(m.VolumeId), math.MaxUint32, math.MaxInt64, dataBaseFileName, ".vif", false, true, nil); err != nil {
				glog.Warningf("ec volume %d: copy .vif from %s: %v", m.VolumeId, peer, err)
			}
			return nil
		})
		if err != nil {
			glog.V(1).Infof("ec volume %d: fetch missing .ecx from %s failed: %v", m.VolumeId, peer, err)
			continue
		}
		glog.V(0).Infof("ec volume %d: fetched missing .ecx from %s into %s", m.VolumeId, peer, m.IdxDir)
		return true
	}
	return false
}