mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-08 15:41:15 +02:00
Chris Lu
2efc0e1656
* ec: recover EC shards whose .ecx index lives only on a peer server A volume server that boots with EC shard files on disk but no .ecx index on any local disk cannot mount the shards, so the master never learns about them. ec.rebuild works off master-registered shards, so it sees the volume as short and gives up even though the shard data is intact. Add an operator-triggered recovery: VolumeEcShardsMount gains a recover_missing_index flag that makes the volume server fetch the missing .ecx (plus .ecj/.vif) from a peer holding it and mount the on-disk shards. ec.rebuild runs this across the cluster before planning, so orphaned shards register and the rebuild sees the true shard set. .ecx is an immutable encode-time index, identical on every holder. .ecj is a per-holder deletion journal that differs across holders, so the recovered node adopts the source peer's deletion view, like a balanced or rebuilt shard does. * ec: mirror missing-index recovery into the Rust volume server Port the #10104 recovery to seaweed-volume so the Rust volume server self-heals the same layout: EC shards on disk with the .ecx index only on a peer. Adds collect_ec_volumes_missing_index / mount_recovered_ec_shards to the store, recover_missing_ec_indexes (master LookupEcVolume + peer CopyFile fetch + mount) to the server, and the recover_missing_index flag on VolumeEcShardsMount. .ecx is the immutable encode-time index, identical on every holder. .ecj is a per-holder deletion journal, so the recovered node adopts the source peer's deletion view, matching the Go path.
84 lines
3.3 KiB
Go
84 lines
3.3 KiB
Go
package storage
|
|
|
|
import (
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
)
|
|
|
|
// EcVolumeMissingIndex names an EC volume whose .ec?? shard files are present on
|
|
// this store but whose .ecx index is missing on every local disk, so the shards
|
|
// cannot be mounted. The per-disk loader and the same-server cross-disk reconcile
|
|
// both leave such shards unmounted because the index lives only on a different
|
|
// server (issue #10104). The recovery path fetches the index from a peer that
|
|
// still holds it and drops it into IdxDir / DataDir, then re-runs the mount.
|
|
type EcVolumeMissingIndex struct {
|
|
Collection string
|
|
VolumeId needle.VolumeId
|
|
IdxDir string // destination for the fetched .ecx / .ecj
|
|
DataDir string // destination for the fetched .vif
|
|
}
|
|
|
|
// CollectEcVolumesMissingIndex returns every EC volume that has shard files on a
|
|
// local disk but no usable .ecx on any local disk — the cross-server orphans a
|
|
// peer index fetch must recover. Volumes whose index merely sits on a sibling
|
|
// disk are excluded (the same-server reconcile/mirror handles those). The
|
|
// destination dirs are taken from the first disk found holding orphan shards;
|
|
// MountRecoveredEcShards mirrors the index onto the remaining shard-bearing disks.
|
|
//
|
|
// It scans on-disk shard files directly, so it surfaces volumes the master never
|
|
// learned about — including those whose every holder is missing its index.
|
|
func (s *Store) CollectEcVolumesMissingIndex() []EcVolumeMissingIndex {
|
|
ecxOwners := s.indexEcxOwners()
|
|
|
|
seen := make(map[ecKeyForReconcile]bool)
|
|
var missing []EcVolumeMissingIndex
|
|
for _, loc := range s.Locations {
|
|
for key := range loc.collectOrphanEcShards() {
|
|
if _, hasLocalIndex := ecxOwners[key]; hasLocalIndex {
|
|
continue
|
|
}
|
|
if seen[key] {
|
|
continue
|
|
}
|
|
seen[key] = true
|
|
missing = append(missing, EcVolumeMissingIndex{
|
|
Collection: key.collection,
|
|
VolumeId: key.vid,
|
|
IdxDir: loc.IdxDirectory,
|
|
DataDir: loc.Directory,
|
|
})
|
|
}
|
|
}
|
|
return missing
|
|
}
|
|
|
|
// MountRecoveredEcShards mounts EC shards that became loadable after a missing
|
|
// .ecx was fetched onto a local disk. It mirrors the index onto every
|
|
// shard-bearing disk, mounts the disks that now have a local index, and falls
|
|
// back to the cross-disk virtual mount for any disk the mirror could not reach.
|
|
// Each shard load announces itself through ecShardNotifyHandler so the master
|
|
// learns about the now-registered shards.
|
|
func (s *Store) MountRecoveredEcShards() {
|
|
s.mirrorEcMetadataToShardDisks()
|
|
s.loadOrphanEcShardsWithLocalIndex()
|
|
s.reconcileEcShardsAcrossDisks()
|
|
}
|
|
|
|
// loadOrphanEcShardsWithLocalIndex mounts on-disk EC shards whose .ecx index is
|
|
// now present on the same disk. Unlike reconcileEcShardsAcrossDisks it does not
|
|
// require a sibling disk, so a single-disk store recovers too once its index has
|
|
// been fetched from a peer.
|
|
func (s *Store) loadOrphanEcShardsWithLocalIndex() {
|
|
for _, loc := range s.Locations {
|
|
orphans := loc.collectOrphanEcShards()
|
|
for key, shards := range orphans {
|
|
if !loc.HasEcxFileOnDisk(key.collection, key.vid) {
|
|
continue
|
|
}
|
|
if err := loc.loadEcShards(shards, key.collection, key.vid, loc.ecShardNotifyHandler); err != nil {
|
|
glog.Errorf("ec volume %d on %s: load after index recovery failed: %v", key.vid, loc.Directory, err)
|
|
}
|
|
}
|
|
}
|
|
}
|