mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 09:46:24 +01:00
* ec: mount shards against the index .vif when the local one has no EC config A worker encode can send shards to a disk that still holds the source volume's .vif while the encode's .vif lands with the index on a sibling disk. The shard disk mounted through the source's config-free .vif, so it reported encode time 0 and fell back to the sidecar layout, and reads carrying the encode time were rejected as a different encode run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): mount shards against the index .vif when the local one has no EC config Mirrors the Go mount: a data-dir .vif that records no EC config yields to the idx dir's when that one records it, and load_vif_info_across_dirs reports the directory the .vif actually came from. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * storage: do not mirror EC sidecars onto a disk whose .vif has no EC config Startup mirroring copied .ecx/.ecj next to a source volume's leftover config-free .vif and kept that .vif, so after a restart the shard disk mounted self-contained with encode time 0. Leave such a disk on the cross-disk mount, which reads the owner's .vif. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): do not mirror EC sidecars onto a disk whose .vif has no EC config Mirrors the Go startup mirror: a shard disk holding a source volume's config-free .vif stays on the cross-disk mount, which reads the owner's .vif, instead of mounting self-contained with encode time 0. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * ec: take the encode .vif from a sibling data dir when the index dir is shared With a separate -dir.idx the .ecx is shared by every disk, but the encode's .vif lands in the data dir of the disk that received it. A disk still holding a source volume's config-free .vif mounted through it at generation 0. NewEcVolume now accepts the server's data dirs and takes a sibling's EC .vif when the index is not local and every sibling EC .vif names the same run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * storage: give EC mounts the data dir of every disk on the server NewStore records every data dir on each DiskLocation before loading, so the startup scan, cross-disk reconcile and MountEcShards all pass them to NewEcVolume. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): take the encode .vif from a sibling data dir when the index dir is shared Mirrors the Go mount: EcVolume::new_with_vif_dirs takes a sibling's EC .vif when this disk's own records no EC config, the index is not local, and every sibling EC .vif names the same run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): give EC mounts the data dir of every disk on the server main sets Store.store_dirs before the first add_location, so a disk loaded before its siblings still sees them; each DiskLocation passes them to EcVolume::new_with_vif_dirs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * storage: size startup EC validation by the sibling encode .vif With a shared -dir.idx, a disk that still holds the source .dat and its config-free .vif read the default data-shard count, judged custom-ratio shards an interrupted encode, and removed them along with the shared .ecx before the mount could take the sibling's .vif. The startup lookup now falls back to the same sibling .vif the mount uses. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): size startup EC validation by the sibling encode .vif Mirrors the Go change: validate_ec_volume falls back to the sibling data dir's encode .vif before judging shards beside a source .dat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * storage: keep EC shards when sibling .vif files disagree on the encode SiblingEcConfigVif reports when a sibling .vif cannot be read or the siblings name different encode runs. The startup data-shard lookup then returns 0 instead of the default ratio, and validateEcVolume keeps the files rather than sizing custom-ratio shards by a ratio they do not use. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * volume(rs): keep EC shards when sibling .vif files disagree on the encode Mirrors the Go change: sibling_ec_config_vif returns Err for an unreadable sibling .vif or disagreeing runs, ec_data_shards_from_vif returns 0 then, and validate_ec_volume keeps the files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * storage: release the opened EC shard when its volume cannot be built Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
814 lines
30 KiB
Go
814 lines
30 KiB
Go
package storage
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"slices"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
|
|
)
|
|
|
|
var (
|
|
// Match .ec00 through .ec999 (currently only .ec00-.ec31 are used)
|
|
// Using \d{2,3} for future-proofing if MaxShardCount is ever increased beyond 99
|
|
re = regexp.MustCompile(`\.ec\d{2,3}`)
|
|
)
|
|
|
|
// markEcDestroying records a pending destroy for vid and returns the channel
|
|
// closed when it finishes. Callers must hold ecVolumesLock for writing.
|
|
func (l *DiskLocation) markEcDestroying(vid needle.VolumeId) chan struct{} {
|
|
if l.ecVolumesDestroying == nil {
|
|
l.ecVolumesDestroying = make(map[needle.VolumeId]chan struct{})
|
|
}
|
|
done := make(chan struct{})
|
|
l.ecVolumesDestroying[vid] = done
|
|
return done
|
|
}
|
|
|
|
func (l *DiskLocation) FindEcVolume(vid needle.VolumeId) (*erasure_coding.EcVolume, bool) {
|
|
l.ecVolumesLock.RLock()
|
|
defer l.ecVolumesLock.RUnlock()
|
|
|
|
ecVolume, ok := l.ecVolumes[vid]
|
|
if ok {
|
|
return ecVolume, true
|
|
}
|
|
return nil, false
|
|
}
|
|
|
|
func (l *DiskLocation) DestroyEcVolume(vid needle.VolumeId) {
|
|
l.ecVolumesLock.Lock()
|
|
ecVolume, found := l.ecVolumes[vid]
|
|
var done chan struct{}
|
|
if found {
|
|
done = l.markEcDestroying(vid)
|
|
delete(l.ecVolumes, vid)
|
|
}
|
|
l.ecVolumesLock.Unlock()
|
|
|
|
if !found {
|
|
return
|
|
}
|
|
// Destroy outside the write lock: EcVolume.Destroy's Close waits on the
|
|
// deletion-journal lock, which a running ec.decode can hold — under the
|
|
// map lock that wait would stall every EC lookup and invert the
|
|
// map->journal lock order.
|
|
ecVolume.Destroy()
|
|
|
|
// The tombstone stays after close() as this vid's destroy generation: a
|
|
// remount compares it before and after opening files to spot a destroy
|
|
// that ran inside the window. The next successful mount clears it.
|
|
close(done)
|
|
}
|
|
|
|
// UnloadEcVolume drops the in-memory EcVolume for vid from this one disk without
|
|
// deleting files. Exported for the generation-fenced teardown, which unloads only
|
|
// the strictly-older disks rather than node-wide.
|
|
func (l *DiskLocation) UnloadEcVolume(vid needle.VolumeId) {
|
|
l.unloadEcVolume(vid)
|
|
}
|
|
|
|
// unloadEcVolume removes an EC volume from memory without deleting its files on disk.
|
|
// This is useful for distributed EC volumes where shards may be on other servers.
|
|
func (l *DiskLocation) unloadEcVolume(vid needle.VolumeId) {
|
|
var toClose *erasure_coding.EcVolume
|
|
l.ecVolumesLock.Lock()
|
|
if ecVolume, found := l.ecVolumes[vid]; found {
|
|
toClose = ecVolume
|
|
delete(l.ecVolumes, vid)
|
|
}
|
|
l.ecVolumesLock.Unlock()
|
|
|
|
// Close outside the lock to avoid holding write lock during I/O
|
|
if toClose != nil {
|
|
toClose.Close()
|
|
}
|
|
}
|
|
|
|
func (l *DiskLocation) CollectEcShards(vid needle.VolumeId, shardFileNames []string) (ecVolume *erasure_coding.EcVolume, found bool) {
|
|
l.ecVolumesLock.RLock()
|
|
defer l.ecVolumesLock.RUnlock()
|
|
|
|
ecVolume, found = l.ecVolumes[vid]
|
|
if !found {
|
|
return
|
|
}
|
|
for _, ecShard := range ecVolume.Shards {
|
|
if ecShard.ShardId < erasure_coding.ShardId(len(shardFileNames)) {
|
|
shardFileNames[ecShard.ShardId] = erasure_coding.EcShardFileName(ecVolume.Collection, l.Directory, int(ecVolume.VolumeId)) + erasure_coding.ToExt(int(ecShard.ShardId))
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
func (l *DiskLocation) FindEcShard(vid needle.VolumeId, shardId erasure_coding.ShardId) (*erasure_coding.EcVolumeShard, bool) {
|
|
l.ecVolumesLock.RLock()
|
|
defer l.ecVolumesLock.RUnlock()
|
|
|
|
ecVolume, ok := l.ecVolumes[vid]
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
for _, ecShard := range ecVolume.Shards {
|
|
if ecShard.ShardId == shardId {
|
|
return ecShard, true
|
|
}
|
|
}
|
|
return nil, false
|
|
}
|
|
|
|
// HasEcxFileOnDisk reports whether this disk has a sealed .ecx index file
|
|
// for the given (collection, vid). Unlike FindEcVolume this does not
|
|
// require the EC volume to be mounted in memory, which makes it the right
|
|
// primitive for placement decisions during ec.balance / ec.rebuild flows
|
|
// where shards may arrive before any mount has happened on the receiving
|
|
// disk. Without checking the on-disk state, auto-select can split shards
|
|
// from the .ecx that travels with the first shard, which is the source of
|
|
// the orphan-shard layout reported in #9212.
|
|
func (l *DiskLocation) HasEcxFileOnDisk(collection string, vid needle.VolumeId) bool {
|
|
// Prefer the local data directory, where the index sits co-located with the
|
|
// shards during a move or reconstruct, then the shared IdxDirectory.
|
|
// A 0-byte .ecx is a corrupt stub left by a failed EC distribute copy;
|
|
// it cannot drive mount and must not steer placement decisions toward
|
|
// this disk. Treat it as absent so the caller falls through to a
|
|
// sibling disk that may hold a valid index.
|
|
dataBase := erasure_coding.EcShardFileName(collection, l.Directory, int(vid))
|
|
if info, err := os.Stat(dataBase + ".ecx"); err == nil && !info.IsDir() && info.Size() > 0 {
|
|
return true
|
|
}
|
|
if l.IdxDirectory != l.Directory {
|
|
idxBase := erasure_coding.EcShardFileName(collection, l.IdxDirectory, int(vid))
|
|
if info, err := os.Stat(idxBase + ".ecx"); err == nil && !info.IsDir() && info.Size() > 0 {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (l *DiskLocation) LoadEcShard(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId) (*erasure_coding.EcVolume, error) {
|
|
return l.loadEcShardWithIdxDir(collection, vid, shardId, l.IdxDirectory)
|
|
}
|
|
|
|
// loadEcShardWithIdxDir is like LoadEcShard but uses the supplied idxDir as
|
|
// the source of .ecx / .ecj rather than this disk's own IdxDirectory. The
|
|
// orphan-shard reconciliation calls this with a sibling disk's idx folder
|
|
// when shards live on a disk that does not own the index files itself
|
|
// (issue #9212).
|
|
func (l *DiskLocation) loadEcShardWithIdxDir(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, idxDir string) (*erasure_coding.EcVolume, error) {
|
|
|
|
// A destroy-in-progress or already-finished one leaves a tombstone; it
|
|
// stays in the map until a mount clears it, so comparing the channel
|
|
// before and after opening the shard detects any destroy that could have
|
|
// unlinked the files in between — including one that ran to completion
|
|
// inside the window.
|
|
var ecVolumeShard *erasure_coding.EcVolumeShard
|
|
for {
|
|
l.ecVolumesLock.Lock()
|
|
gen := l.ecVolumesDestroying[vid]
|
|
l.ecVolumesLock.Unlock()
|
|
if gen != nil {
|
|
select {
|
|
case <-gen:
|
|
// Destroy already finished; the closed tombstone is the
|
|
// generation token the post-open recheck compares against.
|
|
default:
|
|
// Destroy in flight: its unlinks could detach anything this
|
|
// mount opens, so wait it out and take a fresh generation.
|
|
<-gen
|
|
continue
|
|
}
|
|
}
|
|
|
|
var err error
|
|
ecVolumeShard, err = erasure_coding.NewEcVolumeShard(l.DiskType, l.Directory, collection, vid, shardId)
|
|
if err != nil {
|
|
if err == os.ErrNotExist {
|
|
return nil, os.ErrNotExist
|
|
}
|
|
return nil, fmt.Errorf("failed to create ec shard %d.%d: %w", vid, shardId, err)
|
|
}
|
|
|
|
l.ecVolumesLock.Lock()
|
|
if cur := l.ecVolumesDestroying[vid]; cur != gen {
|
|
// A destroy intervened while the shard was being opened; its
|
|
// unlink may have detached the file. Drop the handle, wait for
|
|
// the destroy, and retry on a clean slate.
|
|
l.ecVolumesLock.Unlock()
|
|
ecVolumeShard.Unmount() // release the gauge the constructor's Mount took
|
|
ecVolumeShard.Close()
|
|
if cur != nil {
|
|
<-cur
|
|
}
|
|
continue
|
|
}
|
|
break
|
|
}
|
|
defer l.ecVolumesLock.Unlock()
|
|
ecVolume, found := l.ecVolumes[vid]
|
|
if !found {
|
|
var err error
|
|
ecVolume, err = erasure_coding.NewEcVolume(l.DiskType, l.Directory, idxDir, collection, vid, l.storeDirs...)
|
|
if err != nil {
|
|
ecVolumeShard.Unmount()
|
|
ecVolumeShard.Close()
|
|
// Wrap with %w so MountEcShards / startup reconcile can use
|
|
// errors.Is(err, os.ErrNotExist) to decide whether to try the
|
|
// next local disk vs. surface the failure.
|
|
return nil, fmt.Errorf("failed to create ec volume %d: %w", vid, err)
|
|
}
|
|
l.ecVolumes[vid] = ecVolume
|
|
// Fresh generation for this vid: the next destroy sets a new
|
|
// tombstone rather than matching the cleared one.
|
|
delete(l.ecVolumesDestroying, vid)
|
|
}
|
|
added, err := ecVolume.AddEcVolumeShard(ecVolumeShard)
|
|
if err != nil {
|
|
// The shard could not be registered (e.g. a 0-byte file beside an
|
|
// index with entries). Leave nothing behind: close the opened shard,
|
|
// and remove the EcVolume if this call just created it and it holds
|
|
// no shards — a zero-shard registration would advertise a mount that
|
|
// serves no data while pinning its descriptors.
|
|
ecVolumeShard.Unmount() // release the gauge the constructor's Mount took
|
|
ecVolumeShard.Close()
|
|
if !found && len(ecVolume.Shards) == 0 {
|
|
delete(l.ecVolumes, vid)
|
|
ecVolume.Close()
|
|
}
|
|
return nil, err
|
|
}
|
|
if !added {
|
|
// Already registered on this disk (a mount retry): the existing
|
|
// shard keeps serving; release the duplicate's fd and gauge so
|
|
// repeated LoadEcShard calls don't leak either.
|
|
ecVolumeShard.Unmount()
|
|
ecVolumeShard.Close()
|
|
}
|
|
|
|
return ecVolume, nil
|
|
}
|
|
|
|
// ReopenEcShard swaps the mounted shard's descriptor for a fresh open of its
|
|
// on-disk file while leaving the volume's registration — and therefore the
|
|
// master's view — untouched. It exists for the in-place rebuild path:
|
|
// RebuildEcFiles renames a regenerated shard over the file a mounted handle is
|
|
// still serving, and reopening repoints reads at the repaired file without
|
|
// the delete/add heartbeat flip-flop an unmount+remount would send.
|
|
func (l *DiskLocation) ReopenEcShard(vid needle.VolumeId, shardId erasure_coding.ShardId) (bool, error) {
|
|
l.ecVolumesLock.Lock()
|
|
defer l.ecVolumesLock.Unlock()
|
|
|
|
ecVolume, found := l.ecVolumes[vid]
|
|
if !found {
|
|
return false, nil
|
|
}
|
|
var oldShard *erasure_coding.EcVolumeShard
|
|
position := -1
|
|
for i, s := range ecVolume.Shards {
|
|
if s.ShardId == shardId {
|
|
position = i
|
|
oldShard = s
|
|
break
|
|
}
|
|
}
|
|
if oldShard == nil {
|
|
return false, nil
|
|
}
|
|
|
|
newShard, err := erasure_coding.NewEcVolumeShard(oldShard.DiskType, l.Directory, oldShard.Collection, vid, shardId)
|
|
if err != nil {
|
|
return true, fmt.Errorf("reopen ec shard %d.%d: %w", vid, shardId, err)
|
|
}
|
|
ecVolume.Shards[position] = newShard
|
|
|
|
// Readers that already resolved the old shard keep their object; its Close
|
|
// turns their next ReadAt into a clean error recovered via parity, the same
|
|
// contract eviction relies on. Mount/Unmount balance out, so the shard
|
|
// gauge does not move.
|
|
oldShard.Unmount()
|
|
oldShard.Close()
|
|
return true, nil
|
|
}
|
|
|
|
// MountedEcShardIds returns the ids of shards this disk currently serves for
|
|
// the volume, or nil when the volume is not mounted here.
|
|
func (l *DiskLocation) MountedEcShardIds(vid needle.VolumeId) []erasure_coding.ShardId {
|
|
l.ecVolumesLock.RLock()
|
|
defer l.ecVolumesLock.RUnlock()
|
|
if ecVolume, found := l.ecVolumes[vid]; found {
|
|
return ecVolume.ShardIdList()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *DiskLocation) UnloadEcShard(vid needle.VolumeId, shardId erasure_coding.ShardId) bool {
|
|
|
|
l.ecVolumesLock.Lock()
|
|
defer l.ecVolumesLock.Unlock()
|
|
|
|
ecVolume, found := l.ecVolumes[vid]
|
|
if !found {
|
|
return false
|
|
}
|
|
if _, deleted := ecVolume.DeleteEcVolumeShard(shardId); deleted {
|
|
if len(ecVolume.Shards) == 0 {
|
|
delete(l.ecVolumes, vid)
|
|
ecVolume.Close()
|
|
}
|
|
return true
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func (l *DiskLocation) loadEcShards(shards []string, collection string, vid needle.VolumeId, onShardLoad func(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, ecVolume *erasure_coding.EcVolume)) (err error) {
|
|
|
|
for _, shard := range shards {
|
|
shardId, err := strconv.ParseInt(path.Ext(shard)[3:], 10, 64)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to parse ec shard name %v: %w", shard, err)
|
|
}
|
|
|
|
// Validate shardId range before converting to uint8
|
|
if shardId < 0 || shardId > 255 {
|
|
return fmt.Errorf("shard ID out of range: %d", shardId)
|
|
}
|
|
|
|
ecVolume, err := l.LoadEcShard(collection, vid, erasure_coding.ShardId(shardId))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to load ec shard %v: %w", shard, err)
|
|
}
|
|
if onShardLoad != nil {
|
|
onShardLoad(collection, vid, erasure_coding.ShardId(shardId), ecVolume)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// staleZeroShardAge guards the zero-sized-shard cleanup in loadAllEcShards: a
|
|
// just-created file of an in-flight VolumeEcShardsCopy is legitimately empty
|
|
// for a moment, while failed-operation residue is old by the next scan.
|
|
const staleZeroShardAge = time.Hour
|
|
|
|
func (l *DiskLocation) loadAllEcShards(onShardLoad func(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, ecVolume *erasure_coding.EcVolume)) (err error) {
|
|
|
|
// Keep only the shard and index files this scan acts on: a disk of regular
|
|
// volumes has millions of .dat/.idx/.vif entries that would otherwise each
|
|
// cost a slot in the sorted slice below and a stat() for its size.
|
|
type ecDirEntry struct {
|
|
name string
|
|
size int64
|
|
}
|
|
var dirEntries []ecDirEntry
|
|
collect := func(dir string) error {
|
|
return eachDirEntry(dir, func(entry os.DirEntry) bool {
|
|
if entry.IsDir() {
|
|
return true
|
|
}
|
|
ext := path.Ext(entry.Name())
|
|
if !re.MatchString(ext) && ext != ".ecx" {
|
|
return true
|
|
}
|
|
info, infoErr := entry.Info()
|
|
if infoErr != nil {
|
|
return true
|
|
}
|
|
dirEntries = append(dirEntries, ecDirEntry{name: entry.Name(), size: info.Size()})
|
|
return true
|
|
})
|
|
}
|
|
if err := collect(l.Directory); err != nil {
|
|
return fmt.Errorf("load all ec shards in dir %s: %v", l.Directory, err)
|
|
}
|
|
if l.IdxDirectory != l.Directory {
|
|
if err := collect(l.IdxDirectory); err != nil {
|
|
return fmt.Errorf("load all ec shards in dir %s: %v", l.IdxDirectory, err)
|
|
}
|
|
}
|
|
slices.SortFunc(dirEntries, func(a, b ecDirEntry) int {
|
|
return strings.Compare(a.name, b.name)
|
|
})
|
|
|
|
var sameVolumeShards []string
|
|
var prevVolumeId needle.VolumeId
|
|
var prevCollection string
|
|
|
|
// Helper to reset state between volume processing
|
|
reset := func() {
|
|
sameVolumeShards = nil
|
|
prevVolumeId = 0
|
|
prevCollection = ""
|
|
}
|
|
|
|
for _, fileInfo := range dirEntries {
|
|
name := fileInfo.name
|
|
ext := path.Ext(name)
|
|
baseName := name[:len(name)-len(ext)]
|
|
|
|
collection, volumeId, err := parseCollectionVolumeId(baseName)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
// A zero-sized shard file is residue of a failed operation (never
|
|
// loaded, but its presence poisons later rebuilds, which select
|
|
// inputs from the directory). Delete it once it is old enough that
|
|
// it cannot be an in-flight copy's just-created file — this scan
|
|
// also runs from LoadNewVolumes while the server is serving. The
|
|
// scan merges the Directory and IdxDirectory listings, so the entry
|
|
// and a candidate path can be different files with one name: each
|
|
// candidate's own age decides, and a same-named fresh file (possibly
|
|
// an in-flight copy's just-created one) always survives.
|
|
if re.MatchString(ext) && fileInfo.size == 0 {
|
|
for _, dir := range []string{l.Directory, l.IdxDirectory} {
|
|
p := path.Join(dir, name)
|
|
fi, statErr := os.Stat(p)
|
|
if statErr != nil || fi.IsDir() || fi.Size() != 0 {
|
|
continue
|
|
}
|
|
if time.Since(fi.ModTime()) <= staleZeroShardAge {
|
|
continue
|
|
}
|
|
if rmErr := os.Remove(p); rmErr != nil {
|
|
glog.Warningf("remove zero-sized ec shard %s: %v", p, rmErr)
|
|
} else {
|
|
glog.Warningf("removed zero-sized ec shard %s (residue of a failed operation)", p)
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
|
|
// 0 byte files should be only appearing erroneously for ec data files
|
|
// so we ignore them
|
|
if re.MatchString(ext) && fileInfo.size > 0 {
|
|
// Group shards by both collection and volumeId to avoid mixing collections
|
|
if prevVolumeId == 0 || (volumeId == prevVolumeId && collection == prevCollection) {
|
|
sameVolumeShards = append(sameVolumeShards, name)
|
|
} else {
|
|
// Before starting a new group, check if previous group had orphaned shards
|
|
l.checkOrphanedShards(sameVolumeShards, prevCollection, prevVolumeId)
|
|
sameVolumeShards = []string{name}
|
|
}
|
|
prevVolumeId = volumeId
|
|
prevCollection = collection
|
|
continue
|
|
}
|
|
|
|
if ext == ".ecx" && volumeId == prevVolumeId && collection == prevCollection {
|
|
l.handleFoundEcxFile(sameVolumeShards, collection, volumeId, onShardLoad)
|
|
reset()
|
|
continue
|
|
}
|
|
|
|
}
|
|
|
|
// Check for orphaned EC shards without .ecx file at the end of the directory scan
|
|
// This handles the last group of shards in the directory
|
|
l.checkOrphanedShards(sameVolumeShards, prevCollection, prevVolumeId)
|
|
|
|
return nil
|
|
}
|
|
|
|
// loadEcShardsWithIdxDir loads each shard file in shards into l.ecVolumes,
|
|
// using idxDir as the source of .ecx / .ecj / .vif (NewEcVolume falls back
|
|
// to dirIdx for .vif when the data dir does not have one). Used by the
|
|
// store-level orphan-shard reconciliation in #9212; stops on the first
|
|
// failure so the caller can log and continue with other volumes.
|
|
func (l *DiskLocation) loadEcShardsWithIdxDir(shards []string, collection string, vid needle.VolumeId, idxDir string, onShardLoad func(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, ecVolume *erasure_coding.EcVolume)) error {
|
|
|
|
for _, shard := range shards {
|
|
ext := path.Ext(shard)
|
|
if len(ext) < 4 {
|
|
return fmt.Errorf("unexpected ec shard name %v", shard)
|
|
}
|
|
shardId, err := strconv.ParseInt(ext[3:], 10, 64)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to parse ec shard name %v: %w", shard, err)
|
|
}
|
|
if shardId < 0 || shardId > 255 {
|
|
return fmt.Errorf("shard ID out of range: %d", shardId)
|
|
}
|
|
|
|
ecVolume, err := l.loadEcShardWithIdxDir(collection, vid, erasure_coding.ShardId(shardId), idxDir)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to load ec shard %v: %w", shard, err)
|
|
}
|
|
if onShardLoad != nil {
|
|
onShardLoad(collection, vid, erasure_coding.ShardId(shardId), ecVolume)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (l *DiskLocation) deleteEcVolumeById(vid needle.VolumeId) (e error) {
|
|
l.ecVolumesLock.Lock()
|
|
ecVolume, ok := l.ecVolumes[vid]
|
|
var done chan struct{}
|
|
if ok {
|
|
done = l.markEcDestroying(vid)
|
|
delete(l.ecVolumes, vid)
|
|
}
|
|
l.ecVolumesLock.Unlock()
|
|
|
|
if !ok {
|
|
return
|
|
}
|
|
// Destroy outside the map lock — see DestroyEcVolume.
|
|
ecVolume.Destroy()
|
|
close(done)
|
|
return
|
|
}
|
|
|
|
func (l *DiskLocation) unmountEcVolumeByCollection(collectionName string) map[needle.VolumeId]*erasure_coding.EcVolume {
|
|
deltaVols := make(map[needle.VolumeId]*erasure_coding.EcVolume, 0)
|
|
for k, v := range l.ecVolumes {
|
|
if v.Collection == collectionName {
|
|
deltaVols[k] = v
|
|
}
|
|
}
|
|
|
|
for k := range deltaVols {
|
|
delete(l.ecVolumes, k)
|
|
// Caller destroys these outside the lock; tombstone until then so a
|
|
// remount can't open files the destroy is about to unlink.
|
|
l.markEcDestroying(k)
|
|
}
|
|
return deltaVols
|
|
}
|
|
|
|
func (l *DiskLocation) EcShardCount() int {
|
|
l.ecVolumesLock.RLock()
|
|
defer l.ecVolumesLock.RUnlock()
|
|
|
|
shardCount := 0
|
|
for _, ecVolume := range l.ecVolumes {
|
|
shardCount += len(ecVolume.Shards)
|
|
}
|
|
return shardCount
|
|
}
|
|
|
|
// handleFoundEcxFile processes a complete group of EC shards when their .ecx file is found.
|
|
// This includes validation, loading, and cleanup of incomplete/invalid EC volumes.
|
|
func (l *DiskLocation) handleFoundEcxFile(shards []string, collection string, volumeId needle.VolumeId, onShardLoad func(collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, ecVolume *erasure_coding.EcVolume)) {
|
|
// Check if this is an incomplete EC encoding (not a distributed EC volume)
|
|
// Key distinction: if .dat file still exists, EC encoding may have failed
|
|
// If .dat file is gone, this is likely a distributed EC volume with shards on multiple servers
|
|
baseFileName := erasure_coding.EcShardFileName(collection, l.Directory, int(volumeId))
|
|
datFileName := baseFileName + ".dat"
|
|
|
|
// Determine .dat presence robustly; unexpected errors are treated as "exists"
|
|
datExists := l.checkDatFileExists(datFileName)
|
|
|
|
// Validate EC volume if .dat file exists (incomplete EC encoding scenario)
|
|
// This checks shard count, shard size consistency, and expected size vs .dat file
|
|
// If .dat is gone, EC encoding completed and shards are distributed across servers
|
|
if datExists && !l.validateEcVolume(collection, volumeId) {
|
|
glog.Warningf("Incomplete or invalid EC volume %d: .dat exists but validation failed, cleaning up EC files...", volumeId)
|
|
l.removeEcVolumeFiles(collection, volumeId)
|
|
return
|
|
}
|
|
|
|
// A load failure (corrupt/locked .ecx, EMFILE, transient I/O) is not proof
|
|
// the shards are disposable -- validateEcVolume already decided they may be
|
|
// the only copy. Release FDs but keep the files for retry; never delete here.
|
|
if err := l.loadEcShards(shards, collection, volumeId, onShardLoad); err != nil {
|
|
glog.Warningf("Failed to load EC shards for volume %d: %v; keeping files for retry", volumeId, err)
|
|
l.unloadEcVolume(volumeId)
|
|
return
|
|
}
|
|
}
|
|
|
|
// checkDatFileExists checks if a .dat file with actual data exists with robust
|
|
// error handling. An empty .dat (<= a superblock, zero needles) is a leftover
|
|
// stub, not an encode source, and is treated as absent so it never justifies
|
|
// deleting shards. Unexpected errors (permission, I/O) are treated as "exists"
|
|
// to avoid misclassifying local EC as distributed EC, which is the safer fallback.
|
|
func (l *DiskLocation) checkDatFileExists(datFileName string) bool {
|
|
if fi, err := os.Stat(datFileName); err == nil {
|
|
return fi.Size() > int64(super_block.SuperBlockSize)
|
|
} else if !os.IsNotExist(err) {
|
|
glog.Warningf("Failed to stat .dat file %s: %v", datFileName, err)
|
|
// Safer to assume local .dat exists to avoid misclassifying as distributed EC
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// checkOrphanedShards checks if the given shards are orphaned (no .ecx file) and cleans them up if needed.
|
|
// Returns true if orphaned shards were found and cleaned up.
|
|
// This handles the case where EC encoding was interrupted before creating the .ecx file.
|
|
func (l *DiskLocation) checkOrphanedShards(shards []string, collection string, volumeId needle.VolumeId) bool {
|
|
if len(shards) == 0 || volumeId == 0 {
|
|
return false
|
|
}
|
|
|
|
// Check if .dat file exists (incomplete encoding, not distributed EC)
|
|
baseFileName := erasure_coding.EcShardFileName(collection, l.Directory, int(volumeId))
|
|
datFileName := baseFileName + ".dat"
|
|
|
|
if l.checkDatFileExists(datFileName) {
|
|
glog.Warningf("Found %d EC shards without .ecx file for volume %d (incomplete encoding interrupted before .ecx creation), cleaning up...",
|
|
len(shards), volumeId)
|
|
l.removeEcVolumeFiles(collection, volumeId)
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// calculateExpectedShardSize computes the exact expected shard size based on .dat file size
|
|
// The EC encoding process is deterministic:
|
|
// 1. Data is processed in batches of (LargeBlockSize * dataShardCount) for large blocks
|
|
// 2. Remaining data is processed in batches of (SmallBlockSize * dataShardCount) for small blocks
|
|
// 3. Each shard gets exactly its portion, with zero-padding applied to incomplete blocks
|
|
//
|
|
// dataShardCount is taken as a parameter rather than read from
|
|
// erasure_coding.DataShardsCount so that tests writing a custom layout
|
|
// to .vif compute the matching shard size, and so custom-ratio builds
|
|
// (e.g. enterprise) can swap the default without touching this helper.
|
|
// The padded shard length is the same number under both layouts —
|
|
// TestUniformBlockSizeMatchesLegacyShardSize asserts the equivalence for every
|
|
// input — so defer to the encoder's own helper instead of keeping a second copy
|
|
// of the padding rule that a future change would have to be made in twice. An
|
|
// empty .dat keeps its historic answer: the legacy encoder emits no block for
|
|
// it, where UniformBlockSize floors at one.
|
|
func calculateExpectedShardSize(datFileSize int64, dataShardCount int) int64 {
|
|
if dataShardCount <= 0 || datFileSize <= 0 {
|
|
return 0
|
|
}
|
|
return erasure_coding.UniformBlockSize(datFileSize, dataShardCount)
|
|
}
|
|
|
|
// validateEcVolume reports whether the EC files for (collection, vid) on this
|
|
// disk may be deleted to reclaim the local .dat. It returns false (delete)
|
|
// only when that provably loses no data; every ambiguity returns true (keep),
|
|
// since the shards may be the only copy of distributed-EC data.
|
|
func (l *DiskLocation) validateEcVolume(collection string, vid needle.VolumeId) bool {
|
|
baseFileName := erasure_coding.EcShardFileName(collection, l.Directory, int(vid))
|
|
datFileName := baseFileName + ".dat"
|
|
|
|
// Custom ratio comes from the volume's own .vif; the server holds no
|
|
// cluster EC config in memory.
|
|
dataShards := l.ecDataShardsFromVif(collection, vid)
|
|
if dataShards == 0 {
|
|
glog.Warningf("EC volume %d: sibling .vif files disagree on the encode; keeping EC shards", vid)
|
|
return true
|
|
}
|
|
|
|
// On-disk .dat size, or -1 when absent (an empty <= superblock .dat is a
|
|
// stub). A transient stat error keeps the shards rather than deleting.
|
|
var expectedShardSize int64 = -1
|
|
datExists := false
|
|
if datFileInfo, err := os.Stat(datFileName); err == nil {
|
|
if datFileInfo.Size() > int64(super_block.SuperBlockSize) {
|
|
datExists = true
|
|
expectedShardSize = calculateExpectedShardSize(datFileInfo.Size(), dataShards)
|
|
}
|
|
} else if !os.IsNotExist(err) {
|
|
glog.Warningf("EC volume %d: cannot stat .dat %s (%v); keeping EC shards", vid, datFileName, err)
|
|
return true
|
|
}
|
|
|
|
// Count local shards; a transient stat error or inconsistent sizes -> keep.
|
|
shardCount := 0
|
|
var actualShardSize int64 = -1
|
|
for i := 0; i < erasure_coding.MaxShardCount; i++ {
|
|
shardFileName := baseFileName + erasure_coding.ToExt(i)
|
|
fi, err := os.Stat(shardFileName)
|
|
if err == nil {
|
|
if fi.Size() > 0 {
|
|
if actualShardSize == -1 {
|
|
actualShardSize = fi.Size()
|
|
} else if fi.Size() != actualShardSize {
|
|
glog.Warningf("EC volume %d shard %d size %d != %d; keeping EC shards", vid, i, fi.Size(), actualShardSize)
|
|
return true
|
|
}
|
|
shardCount++
|
|
}
|
|
} else if !os.IsNotExist(err) {
|
|
glog.Warningf("EC volume %d: cannot stat shard %s (%v); keeping EC shards", vid, shardFileName, err)
|
|
return true
|
|
}
|
|
}
|
|
|
|
if !datExists {
|
|
return true // distributed EC; any shard count is valid
|
|
}
|
|
|
|
// Reclaim only when it loses no data. Shards smaller than this .dat's full
|
|
// encode are an interrupted encode whose .dat is the complete source ->
|
|
// reclaim. Shards >= expected (valid/distributing EC, or a stale/partial
|
|
// .dat beside larger real shards) may be the only copy -> keep.
|
|
if shardCount == 0 {
|
|
return false
|
|
}
|
|
if expectedShardSize > 0 && actualShardSize > 0 && actualShardSize < expectedShardSize {
|
|
glog.Warningf("EC volume %d: %d shards of %d bytes are smaller than the .dat's full encode (%d bytes); reclaiming the complete .dat",
|
|
vid, shardCount, actualShardSize, expectedShardSize)
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// ecDataShardsFromVif resolves the data-shard count for an EC volume from
|
|
// its own .vif (EcShardConfig), checking the data dir, the idx dir, then the
|
|
// sibling data dir the mount takes the .vif from under a shared idx dir. The
|
|
// .vif is the source of truth for custom ratios on the volume server, which
|
|
// never holds the cluster EC config in memory. Falls back to the default
|
|
// ratio when the .vif carries no EC shard config.
|
|
func (l *DiskLocation) ecDataShardsFromVif(collection string, vid needle.VolumeId) int {
|
|
// At most two dirs to check; avoid slice/map allocations on this
|
|
// per-volume startup path.
|
|
if l.Directory != "" {
|
|
if ds := ecDataShardsFromVifDir(collection, l.Directory, vid); ds > 0 {
|
|
return ds
|
|
}
|
|
}
|
|
if l.IdxDirectory != "" && l.IdxDirectory != l.Directory {
|
|
if ds := ecDataShardsFromVifDir(collection, l.IdxDirectory, vid); ds > 0 {
|
|
return ds
|
|
}
|
|
vif, ambiguous := erasure_coding.SiblingEcConfigVif(collection, vid, l.Directory, l.IdxDirectory, l.storeDirs)
|
|
if ambiguous {
|
|
return 0
|
|
}
|
|
if vif != "" {
|
|
if ds := ecDataShardsFromVifPath(vif); ds > 0 {
|
|
return ds
|
|
}
|
|
}
|
|
}
|
|
return erasure_coding.DataShardsCount
|
|
}
|
|
|
|
// ecDataShardsFromVifDir returns the .vif EcShardConfig data-shard count for
|
|
// (collection, vid) under dir, or 0 when absent / not custom.
|
|
func ecDataShardsFromVifDir(collection, dir string, vid needle.VolumeId) int {
|
|
return ecDataShardsFromVifPath(erasure_coding.EcShardFileName(collection, dir, int(vid)) + ".vif")
|
|
}
|
|
|
|
// ecDataShardsFromVifPath returns the EcShardConfig data-shard count recorded
|
|
// in one .vif, or 0 when absent / not custom.
|
|
func ecDataShardsFromVifPath(vifName string) int {
|
|
if vi, _, found, _ := volume_info.MaybeLoadVolumeInfo(vifName); found && vi.EcShardConfig != nil {
|
|
if ds := int(vi.EcShardConfig.DataShards); ds > 0 {
|
|
return ds
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// removeEcVolumeFiles removes all EC-related files for a volume
|
|
func (l *DiskLocation) removeEcVolumeFiles(collection string, vid needle.VolumeId) {
|
|
baseFileName := erasure_coding.EcShardFileName(collection, l.Directory, int(vid))
|
|
indexBaseFileName := erasure_coding.EcShardFileName(collection, l.IdxDirectory, int(vid))
|
|
|
|
// Helper to remove a file with consistent error handling
|
|
removeFile := func(filePath, description string) {
|
|
if err := os.Remove(filePath); err != nil {
|
|
if !os.IsNotExist(err) {
|
|
glog.Warningf("Failed to remove incomplete %s %s: %v", description, filePath, err)
|
|
}
|
|
} else {
|
|
glog.V(2).Infof("Removed incomplete %s: %s", description, filePath)
|
|
}
|
|
}
|
|
|
|
// Remove index files first (.ecx, .ecj) before shard files
|
|
// This ensures that if cleanup is interrupted, the .ecx file won't trigger
|
|
// EC loading for incomplete/missing shards on next startup
|
|
removeFile(indexBaseFileName+".ecx", "EC index file")
|
|
removeFile(indexBaseFileName+".ecj", "EC journal file")
|
|
removeFile(indexBaseFileName+erasure_coding.EcjCompactTmpExt, "EC journal compaction tmp")
|
|
// Also try the data directory in case .ecx/.ecj were created before -dir.idx was configured
|
|
if l.IdxDirectory != l.Directory {
|
|
removeFile(baseFileName+".ecx", "EC index file (fallback)")
|
|
removeFile(baseFileName+".ecj", "EC journal file (fallback)")
|
|
removeFile(baseFileName+erasure_coding.EcjCompactTmpExt, "EC journal compaction tmp (fallback)")
|
|
}
|
|
|
|
// Remove all EC shard files (.ec00 ~ .ec31) from data directory
|
|
// Use MaxShardCount (32) to support custom EC ratios
|
|
for i := 0; i < erasure_coding.MaxShardCount; i++ {
|
|
removeFile(baseFileName+erasure_coding.ToExt(i), "EC shard file")
|
|
}
|
|
|
|
// Staged 2PC generations (<base>.ecNN.v<N>, versioned .ecx/.ecj/.vif)
|
|
// belong to this volume's EC state too; leaving them orphans the files.
|
|
for _, dir := range []string{indexBaseFileName, baseFileName} {
|
|
if err := erasure_coding.RemoveEcGenerationFiles(dir, 0); err != nil {
|
|
glog.Warningf("Failed to remove EC generation files for %s: %v", dir, err)
|
|
}
|
|
}
|
|
}
|