seaweedfs/weed/filer/filer_deletion_persist.go

package filer

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

// Persistent deletion ledger.
//
// Background: the in-memory FileIdDeletionQueue and DeletionRetryQueue lose every
// queued-but-unconfirmed deletion when the filer process restarts (the upstream
// TODO in filer_deletion.go notes this and proposes exactly the "periodic snapshot
// with recovery on startup" strategy implemented here). Because a delete only ever
// enters the pipeline through the queue, a crash between enqueue and the volume
// actually confirming the delete leaks the chunk: nothing remembers it, so it
// becomes an orphan the next fsck sees as 100% orphaned and a later
// meta-replay from a lagging peer can "resurrect" as if it were live data.
//
// The ledger is a KV entry per filer (KvKeyDeletionLedger suffixed with this
// filer's address, so filers sharing one store never overwrite each other's
// pending sets) holding the fileIds that still need to be deleted but have not
// yet been confirmed gone. It is:
//   - additive: every EnQueue also records the fileId here,
//   - subtractive: only terminal outcomes (success / not-found / permanent)
//     remove it; retryable failures keep it,
//   - snapshotted when it changes, on a timer, and on Shutdown, so a crash
//     loses only ids queued in the last in-flight write — recovery re-enqueues
//     the whole pending set and the idempotent volume delete absorbs anything
//     that actually completed before the crash.
//
// Stores that cap value size (FoundationDB at 100KB) get the set split into
// part keys when one value would exceed deletionLedgerPartSize. Multipart
// snapshots write each generation under generation-scoped part keys and
// publish only the manifest last, so a crash mid-write leaves the previous
// generation intact; orphaned parts from abandoned generations are tracked in
// deletionLedgerStale and retried on the next write.
//
// Safety on a lagging peer (the resurrection case): recovered entries are replayed
// into the queue only after a grace window (DeletionRecoveryGrace), long enough for
// the initial peer meta-aggregation to settle so we don't purge a chunk that a
// peer is about to re-reference as live. New live deletes are never gated.
const (
	// KvKeyDeletionLedger is the reserved store key prefix for the persisted
	// ledger, namespaced next to FilerStoreId so it never collides with user
	// data. The full key is this prefix plus the filer's own address.
	KvKeyDeletionLedger = "filer.deleteQueue.ledger.v1"

	// KvKeyDeletionLedgerIndex lists the scoped ledger keys in the store so a
	// filer that restarts under a different advertised address can still find
	// and claim the ledger it left behind.
	KvKeyDeletionLedgerIndex = KvKeyDeletionLedger + ".index"

	// deletionLedgerPartSize bounds one ledger value; stores with a value cap
	// reject anything larger, which would strand a big backlog in memory only.
	deletionLedgerPartSize = 64 * 1024

	// Defaults; all three overridable via viper (filer.deleteQueue.*).
	defaultDeletionPersistInterval = 10 * time.Second
	defaultDeletionRecoveryGrace   = 30 * time.Second
)

// deletionPersistEnabled reports whether ledger persistence is on.
// Read from viper with a default of true so a plain `weed filer` gets the
// durability guarantee without extra flags.
func deletionPersistEnabled() bool {
	v := util.GetViper()
	v.SetDefault("filer.deleteQueue.persist", true)
	return v.GetBool("filer.deleteQueue.persist")
}

func deletionPersistInterval() time.Duration {
	v := util.GetViper()
	v.SetDefault("filer.deleteQueue.persistInterval", defaultDeletionPersistInterval)
	d := v.GetDuration("filer.deleteQueue.persistInterval")
	if d <= 0 {
		return defaultDeletionPersistInterval
	}
	return d
}

func deletionRecoveryGrace() time.Duration {
	v := util.GetViper()
	v.SetDefault("filer.deleteQueue.recoveryGrace", defaultDeletionRecoveryGrace)
	d := v.GetDuration("filer.deleteQueue.recoveryGrace")
	if d < 0 {
		return 0
	}
	return d
}

// deletionLedgerKey scopes the ledger to this filer so several filers sharing
// one metadata store do not overwrite each other's pending sets. A filer
// literal without an advertised address (tests) falls back to the base key.
func (f *Filer) deletionLedgerKey() string {
	if f.Dlm != nil && f.Dlm.Host != "" {
		return KvKeyDeletionLedger + "." + f.Dlm.Host.ToHttpAddress()
	}
	return KvKeyDeletionLedger
}

func deletionLedgerPartKey(key string, part int) []byte {
	return []byte(fmt.Sprintf("%s.part.%05d", key, part))
}

// deletionLedgerGenPartKey returns the part key for one generation. Generation
// 0 keeps the original format so ledgers written before generation publishing
// still decode.
func deletionLedgerGenPartKey(key string, gen, part int) []byte {
	if gen == 0 {
		return deletionLedgerPartKey(key, part)
	}
	return []byte(fmt.Sprintf("%s.g%06d.part.%05d", key, gen, part))
}

// startDeletionLedgerSnapshotter periodically flushes the pending deletion set
// to durable storage. It also wakes on deletionLedgerFlush so a queued id is
// persisted within milliseconds instead of a full interval. Started from
// SetStore only after the ledger read succeeded.
func (f *Filer) startDeletionLedgerSnapshotter() {
	go func() {
		ticker := time.NewTicker(deletionPersistInterval())
		defer ticker.Stop()
		for {
			select {
			case <-f.deletionQuit:
				return
			case <-ticker.C:
			case <-f.deletionLedgerFlush:
			}
			// A close of deletionQuit may be concurrent with this wake; the
			// final snapshot in Shutdown covers whatever the loop left dirty.
			select {
			case <-f.deletionQuit:
				return
			default:
			}
			f.snapshotDeletionLedger()
		}
	}()
}

// signalLedgerFlush wakes the snapshotter without blocking the caller.
func (f *Filer) signalLedgerFlush() {
	f.deletionLedgerLock.Lock()
	if f.deletionLedgerFlush == nil {
		f.deletionLedgerFlush = make(chan struct{}, 1)
	}
	ch := f.deletionLedgerFlush
	f.deletionLedgerLock.Unlock()
	select {
	case ch <- struct{}{}:
	default:
	}
}

// queueDeletions is the single entry point for adding fileIds to the deletion
// pipeline. It keeps the in-memory hot queue AND the durable ledger in sync.
// Safe on a zero-value Filer (tests build struct literals without NewFiler):
// the map is lazily created and the mutex is zero-value friendly.
func (f *Filer) queueDeletions(fileIds ...string) {
	if len(fileIds) == 0 {
		return
	}
	// Hot path: existing in-memory queue, unchanged.
	f.FileIdDeletionQueue.EnQueue(fileIds...)

	// Durable ledger: record what still needs deleting. Every enqueue bumps the
	// id's epoch so an expiry-forget carrying an older epoch cannot erase the
	// re-queued intent.
	f.deletionLedgerLock.Lock()
	if f.pendingDeletions == nil {
		f.pendingDeletions = make(map[string]uint64, len(fileIds))
	}
	for _, id := range fileIds {
		if id == "" {
			continue
		}
		_, exists := f.pendingDeletions[id]
		f.deletionSeq++
		f.pendingDeletions[id] = f.deletionSeq
		if !exists {
			f.deletionLedgerDirty = true
		}
	}
	f.deletionLedgerLock.Unlock()
	f.signalLedgerFlush()
}

// pendingDeletionCount returns the number of fileIds still tracked as needing
// deletion in the durable ledger. Handy for diagnostics and tests; 0 when the
// map was never initialised.
func (f *Filer) pendingDeletionCount() int {
	f.deletionLedgerLock.Lock()
	defer f.deletionLedgerLock.Unlock()
	return len(f.pendingDeletions)
}

// deletionEpoch returns the pending record's current epoch for fileId.
func (f *Filer) deletionEpoch(fileId string) uint64 {
	f.deletionLedgerLock.Lock()
	defer f.deletionLedgerLock.Unlock()
	return f.pendingDeletions[fileId]
}

// forgetDeletion removes a fileId from the ledger once its deletion is terminal
// (deleted or already absent: the chunks are gone, so any pending record — even
// a re-queued one — refers to work that no longer exists). Non-terminal
// outcomes must NOT call this so the entry survives until confirmed.
func (f *Filer) forgetDeletion(fileId string) {
	if fileId == "" {
		return
	}
	f.deletionLedgerLock.Lock()
	if _, exists := f.pendingDeletions[fileId]; exists {
		delete(f.pendingDeletions, fileId)
		f.deletionLedgerDirty = true
	}
	f.deletionLedgerLock.Unlock()
	f.signalLedgerFlush()
}

// forgetDeletionEpoch removes the ledger record only when its epoch still
// matches the one captured when the deletion attempt began. A mismatch means
// the id was re-queued meanwhile, so the newer record wins. Reports whether
// the record was removed.
func (f *Filer) forgetDeletionEpoch(fileId string, epoch uint64) bool {
	if fileId == "" {
		return false
	}
	removed := false
	f.deletionLedgerLock.Lock()
	if cur, exists := f.pendingDeletions[fileId]; exists && cur == epoch {
		delete(f.pendingDeletions, fileId)
		f.deletionLedgerDirty = true
		removed = true
	}
	f.deletionLedgerLock.Unlock()
	if removed {
		f.signalLedgerFlush()
	}
	return removed
}

// snapshotDeletionLedger serialises the current pending set to the store.
// Only writes when something changed since the last snapshot to keep KV churn low.
// Writes serialize on deletionSnapshotLock so a snapshot in flight when Shutdown
// starts cannot overwrite the final one with an older copy.
func (f *Filer) snapshotDeletionLedger() {
	if !deletionPersistEnabled() || f.Store == nil {
		return
	}
	f.deletionSnapshotLock.Lock()
	defer f.deletionSnapshotLock.Unlock()

	if f.deletionLedgerBlocked.Load() && !f.tryUnblockDeletionLedger() {
		return
	}

	f.deletionLedgerLock.Lock()
	if !f.deletionLedgerDirty {
		hasStale := len(f.deletionLedgerStale) > 0
		f.deletionLedgerLock.Unlock()
		if !hasStale {
			return
		}
		// Nothing to republish, but orphaned part keys still need cleanup.
		ctx := context.Background()
		f.flushStaleLedgerParts(ctx)
		f.persistStaleLedgerParts(ctx, f.deletionLedgerKey())
		return
	}
	// Take a stable copy under lock; marshal + KV write happen outside it.
	ids := make([]string, 0, len(f.pendingDeletions))
	for id := range f.pendingDeletions {
		ids = append(ids, id)
	}
	f.deletionLedgerDirty = false
	f.deletionLedgerLock.Unlock()

	if err := f.writeDeletionLedger(ids); err != nil {
		glog.Warningf("failed to persist deletion ledger (%d ids): %v", len(ids), err)
		f.deletionLedgerLock.Lock()
		f.deletionLedgerDirty = true
		f.deletionLedgerLock.Unlock()
		return
	}
	glog.V(3).Infof("persisted deletion ledger: %d pending deletions", len(ids))
}

// tryUnblockDeletionLedger retries the ledger reload that originally failed.
// Once the read succeeds the persisted ids merge back into the pending set and
// snapshots resume; until then nothing is written so the unread ledger cannot
// be overwritten.
func (f *Filer) tryUnblockDeletionLedger() bool {
	f.reloadDeletionLedger()
	return !f.deletionLedgerBlocked.Load()
}

// writeDeletionLedger persists the id set. A single value is published
// atomically; a multipart set writes a new generation's part keys first and
// only then the manifest, so a crash or failed write leaves the previously
// published generation readable. Orphaned parts are retried on the next write.
func (f *Filer) writeDeletionLedger(ids []string) error {
	ctx := context.Background()
	key := f.deletionLedgerKey()

	f.deletionLedgerLock.Lock()
	prevGen, prevParts := f.deletionLedgerGen, f.deletionLedgerParts
	f.deletionLedgerLock.Unlock()

	f.flushStaleLedgerParts(ctx)

	var parts [][]byte
	var batch []string
	size := 2 // "[]"
	flush := func() error {
		if batch == nil {
			batch = []string{}
		}
		payload, err := json.Marshal(batch)
		if err != nil {
			return err
		}
		parts = append(parts, payload)
		batch = nil
		size = 2
		return nil
	}
	for _, id := range ids {
		if need := len(id) + 3; len(batch) > 0 && size+need > deletionLedgerPartSize {
			if err := flush(); err != nil {
				return err
			}
		}
		batch = append(batch, id)
		size += len(id) + 3
	}
	if len(batch) > 0 || len(parts) == 0 {
		if err := flush(); err != nil {
			return err
		}
	}

	if len(parts) == 1 {
		if err := f.Store.KvPut(ctx, []byte(key), parts[0]); err != nil {
			return err
		}
		f.deleteLedgerParts(ctx, key, prevGen, prevParts)
		f.deletionLedgerLock.Lock()
		f.deletionLedgerGen, f.deletionLedgerParts = 0, 0
		f.deletionLedgerLock.Unlock()
	} else {
		gen := prevGen + 1
		wrote := 0
		for i, payload := range parts {
			if err := f.Store.KvPut(ctx, deletionLedgerGenPartKey(key, gen, i), payload); err != nil {
				f.markStaleLedgerParts(key, gen, wrote)
				return err
			}
			wrote++
		}
		manifest, _ := json.Marshal(struct {
			Parts int `json:"parts"`
			Gen   int `json:"gen,omitempty"`
		}{len(parts), gen})
		if err := f.Store.KvPut(ctx, []byte(key), manifest); err != nil {
			f.markStaleLedgerParts(key, gen, wrote)
			return err
		}
		f.deleteLedgerParts(ctx, key, prevGen, prevParts)
		f.deletionLedgerLock.Lock()
		f.deletionLedgerGen, f.deletionLedgerParts = gen, len(parts)
		f.deletionLedgerLock.Unlock()
	}
	f.persistStaleLedgerParts(ctx, key)
	f.touchLedgerIndex(ctx, key)
	return nil
}

// deleteLedgerParts removes a published generation's part keys; failures are
// tracked as stale so the next snapshot retries them.
func (f *Filer) deleteLedgerParts(ctx context.Context, key string, gen, parts int) {
	for i := 0; i < parts; i++ {
		if err := f.Store.KvDelete(ctx, deletionLedgerGenPartKey(key, gen, i)); err != nil {
			f.deletionLedgerLock.Lock()
			f.deletionLedgerStale = append(f.deletionLedgerStale, string(deletionLedgerGenPartKey(key, gen, i)))
			f.deletionLedgerLock.Unlock()
		}
	}
}

// markStaleLedgerParts remembers parts of an abandoned generation so they are
// deleted by a later snapshot instead of leaking store space.
func (f *Filer) markStaleLedgerParts(key string, gen, wrote int) {
	if wrote == 0 {
		return
	}
	f.deletionLedgerLock.Lock()
	for i := 0; i < wrote; i++ {
		f.deletionLedgerStale = append(f.deletionLedgerStale, string(deletionLedgerGenPartKey(key, gen, i)))
	}
	f.deletionLedgerLock.Unlock()
}

func (f *Filer) flushStaleLedgerParts(ctx context.Context) {
	f.deletionLedgerLock.Lock()
	stale := f.deletionLedgerStale
	f.deletionLedgerStale = nil
	f.deletionLedgerLock.Unlock()

	var keep []string
	for _, k := range stale {
		if err := f.Store.KvDelete(ctx, []byte(k)); err != nil {
			keep = append(keep, k)
		}
	}
	if len(keep) > 0 {
		f.deletionLedgerLock.Lock()
		f.deletionLedgerStale = append(keep, f.deletionLedgerStale...)
		f.deletionLedgerLock.Unlock()
	}
}

// persistStaleLedgerParts mirrors the stale-part list under a sidecar key so a
// crash between manifest publication and part cleanup can retry after restart
// instead of orphaning the part keys.
func (f *Filer) persistStaleLedgerParts(ctx context.Context, key string) {
	f.deletionLedgerLock.Lock()
	stale := append([]string(nil), f.deletionLedgerStale...)
	f.deletionLedgerLock.Unlock()
	staleKey := []byte(key + ".stale")
	if len(stale) == 0 {
		_ = f.Store.KvDelete(ctx, staleKey)
		return
	}
	payload, _ := json.Marshal(stale)
	_ = f.Store.KvPut(ctx, staleKey, payload)
}

// loadStaleLedgerParts seeds the stale-part list left by a previous run.
func (f *Filer) loadStaleLedgerParts(ctx context.Context, key string) {
	payload, err := f.Store.KvGet(ctx, []byte(key+".stale"))
	if err != nil {
		return
	}
	var stale []string
	if json.Unmarshal(payload, &stale) != nil {
		return
	}
	f.deletionLedgerLock.Lock()
	f.deletionLedgerStale = append(f.deletionLedgerStale, stale...)
	f.deletionLedgerLock.Unlock()
}

// touchLedgerIndex records this filer's scoped key in the shared index so the
// ledger remains discoverable if the filer restarts under a new address. The
// index is a read-modify-write list with no CAS, so the write is verified and
// retried: a peer's concurrent update must not drop this key.
func (f *Filer) touchLedgerIndex(ctx context.Context, key string) {
	if key == KvKeyDeletionLedger {
		return
	}
	for attempt := 0; attempt < 3; attempt++ {
		keys, err := f.readLedgerIndex(ctx)
		if err != nil {
			// Only a genuinely absent index means "empty"; a store error must
			// not let us write a one-key list that drops every peer entry.
			if err != ErrKvNotFound {
				glog.V(1).Infof("deletion ledger index unreadable, skipping update: %v", err)
				return
			}
			keys = nil
		}
		found := false
		for _, k := range keys {
			if k == key {
				found = true
				break
			}
		}
		if found {
			return
		}
		payload, _ := json.Marshal(append(keys, key))
		if err := f.Store.KvPut(ctx, []byte(KvKeyDeletionLedgerIndex), payload); err != nil {
			glog.V(1).Infof("failed to update deletion ledger index: %v", err)
			return
		}
	}
	glog.V(1).Infof("deletion ledger index lost %q to a concurrent update; retrying on the next snapshot", key)
}

func (f *Filer) readLedgerIndex(ctx context.Context) ([]string, error) {
	payload, err := f.Store.KvGet(ctx, []byte(KvKeyDeletionLedgerIndex))
	if err != nil {
		return nil, err
	}
	var keys []string
	if err := json.Unmarshal(payload, &keys); err != nil {
		return nil, err
	}
	return keys, nil
}

func (f *Filer) pruneLedgerIndex(ctx context.Context, key string) {
	keys, err := f.readLedgerIndex(ctx)
	if err != nil {
		return
	}
	kept := keys[:0]
	for _, k := range keys {
		if k != key {
			kept = append(kept, k)
		}
	}
	payload, _ := json.Marshal(kept)
	_ = f.Store.KvPut(ctx, []byte(KvKeyDeletionLedgerIndex), payload)
}

// readDeletionLedger reads the manifest key: a JSON array is the whole set; a
// {"parts":N,"gen":G} manifest points at that generation's part keys. A part
// the manifest references but the store lacks is corruption, not absence, so
// it surfaces as a wrapped error rather than ErrKvNotFound.
func (f *Filer) readDeletionLedger(key string) (ids []string, gen, parts int, err error) {
	ctx := context.Background()

	payload, err := f.Store.KvGet(ctx, []byte(key))
	if err != nil {
		return nil, 0, 0, err
	}
	if !bytes.HasPrefix(bytes.TrimSpace(payload), []byte("{")) {
		if err := json.Unmarshal(payload, &ids); err != nil {
			return nil, 0, 0, err
		}
		return ids, 0, 0, nil
	}
	var manifest struct {
		Parts int `json:"parts"`
		Gen   int `json:"gen,omitempty"`
	}
	if err := json.Unmarshal(payload, &manifest); err != nil {
		return nil, 0, 0, err
	}
	for i := 0; i < manifest.Parts; i++ {
		partPayload, err := f.Store.KvGet(ctx, deletionLedgerGenPartKey(key, manifest.Gen, i))
		if err != nil {
			return nil, 0, 0, fmt.Errorf("deletion ledger part %d of %s unreadable: %w", i, key, err)
		}
		var part []string
		if err := json.Unmarshal(partPayload, &part); err != nil {
			return nil, 0, 0, fmt.Errorf("deletion ledger part %d of %s corrupt: %w", i, key, err)
		}
		ids = append(ids, part...)
	}
	return ids, manifest.Gen, manifest.Parts, nil
}

// recoverForeignDeletionLedgers runs when the scoped key is absent: the ledger
// may sit under the pre-scoping base key, or under a scoped key belonging to an
// earlier incarnation of this filer whose advertised address changed. Every
// discovered set is published under this filer's key first and the source keys
// deleted only after that write succeeds. A live peer's ledger claimed here is
// rewritten by the peer's next snapshot, so the ids only get processed twice —
// deletions are not owner-specific.
func (f *Filer) recoverForeignDeletionLedgers(key string) (ids []string, err error) {
	ctx := context.Background()

	claimed := map[string]ledgerClaim{} // source key -> what was read
	lIds, lGen, lParts, lErr := f.readDeletionLedger(KvKeyDeletionLedger)
	switch {
	case lErr == nil:
		ids = append(ids, lIds...)
		claimed[KvKeyDeletionLedger] = ledgerClaim{lIds, lGen, lParts}
		f.loadStaleLedgerParts(ctx, KvKeyDeletionLedger)
	case lErr != ErrKvNotFound:
		return nil, lErr
	}

	// Any read failure aborts the whole claim so reload retries: skipping an
	// unreadable source would strand it, since a successful claim here means
	// the index is never searched again.
	indexKeys, idxErr := f.readLedgerIndex(ctx)
	if idxErr != nil && idxErr != ErrKvNotFound {
		return nil, idxErr
	}
	for _, other := range indexKeys {
		if other == key {
			continue
		}
		oIds, oGen, oParts, oErr := f.readDeletionLedger(other)
		if oErr == ErrKvNotFound {
			f.pruneLedgerIndex(ctx, other)
			continue
		}
		if oErr != nil {
			return nil, fmt.Errorf("deletion ledger %s unreadable: %w", other, oErr)
		}
		ids = append(ids, oIds...)
		claimed[other] = ledgerClaim{oIds, oGen, oParts}
		f.loadStaleLedgerParts(ctx, other)
	}

	if len(claimed) == 0 {
		return nil, ErrKvNotFound
	}
	if err := f.writeDeletionLedger(ids); err != nil {
		return nil, err
	}
	mergedMore := false
	for src, claim := range claimed {
		// A source republished since we read it belongs to a live filer
		// (or a racing claimer): merge the newer ids and leave it in place.
		curIds, curGen, curParts, rerr := f.readDeletionLedger(src)
		if rerr != nil || curGen != claim.gen || curParts != claim.parts || !sameStringSet(curIds, claim.ids) {
			if rerr == nil {
				for _, id := range curIds {
					if !containsId(ids, id) {
						ids = append(ids, id)
						mergedMore = true
					}
				}
			}
			glog.V(0).Infof("deletion ledger %s changed while claiming; leaving it for its owner", src)
			continue
		}
		f.deleteLedgerParts(ctx, src, claim.gen, claim.parts)
		_ = f.Store.KvDelete(ctx, []byte(src))
		_ = f.Store.KvDelete(ctx, []byte(src+".stale"))
		f.pruneLedgerIndex(ctx, src)
	}
	if mergedMore {
		// Ids from a changed source are durable only under that source's key;
		// rewrite our ledger so they survive under ours too.
		if err := f.writeDeletionLedger(ids); err != nil {
			return nil, err
		}
	}
	return ids, nil
}

func containsId(ids []string, id string) bool {
	for _, s := range ids {
		if s == id {
			return true
		}
	}
	return false
}

type ledgerClaim struct {
	ids   []string
	gen   int
	parts int
}

func sameStringSet(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	seen := make(map[string]struct{}, len(a))
	for _, s := range a {
		seen[s] = struct{}{}
	}
	for _, s := range b {
		if _, ok := seen[s]; !ok {
			return false
		}
	}
	return true
}

// reloadDeletionLedger re-enqueues any pending deletions found in the store after
// a restart, so a crash that killed the in-memory queues does not leak chunks.
// Recovered ids join the pending set immediately so snapshots rewrite the full
// ledger; only the re-queue waits out DeletionRecoveryGrace.
//
// It reports whether the ledger is usable. A read error or an unparseable
// payload leaves the persisted set unknown, so snapshots retry the read rather
// than overwrite the unread ledger with a partial set.
//
// Safe to call on a Filer with a nil Store (no-op). Idempotent for the volume
// side: a chunk that was actually deleted before the crash re-deletes as not-found.
func (f *Filer) reloadDeletionLedger() bool {
	if !deletionPersistEnabled() || f.Store == nil {
		return false
	}

	key := f.deletionLedgerKey()
	ids, gen, parts, err := f.readDeletionLedger(key)
	stateWritten := false
	if err == ErrKvNotFound && key != KvKeyDeletionLedger {
		ids, err = f.recoverForeignDeletionLedgers(key)
		stateWritten = err == nil
	}
	if err != nil {
		if err == ErrKvNotFound {
			f.deletionLedgerBlocked.Store(false)
			return true
		}
		f.deletionLedgerBlocked.Store(true)
		glog.Warningf("failed to read persisted deletion ledger; persistence disabled until it reads: %v", err)
		return false
	}
	f.deletionLedgerBlocked.Store(false)
	f.loadStaleLedgerParts(context.Background(), key)

	// Merge into the pending set now so an early snapshot rewrites the
	// recovered ids instead of overwriting the ledger with only new ones.
	f.deletionLedgerLock.Lock()
	if len(ids) > 0 {
		if f.pendingDeletions == nil {
			f.pendingDeletions = make(map[string]uint64, len(ids))
		}
		for _, id := range ids {
			if id != "" {
				if _, exists := f.pendingDeletions[id]; !exists {
					f.deletionSeq++
					f.pendingDeletions[id] = f.deletionSeq
				}
			}
		}
	}
	if !stateWritten {
		f.deletionLedgerGen = gen
		f.deletionLedgerParts = parts
	}
	f.deletionLedgerLock.Unlock()

	if len(ids) == 0 {
		return true
	}

	grace := deletionRecoveryGrace()
	glog.V(0).Infof("recovered %d pending deletions from ledger, applying in %v", len(ids), grace)

	go func() {
		timer := time.NewTimer(grace)
		defer timer.Stop()
		select {
		case <-f.deletionQuit:
			return
		case <-timer.C:
		}
		// The ids are already in the pending set; only the queue push waits
		// for peer meta-aggregation to settle. The ledger is deliberately NOT
		// cleared here: it shrinks only as the delete pipeline confirms each
		// id terminal, so a second crash mid-recovery replays everything.
		f.queueDeletions(ids...)
		glog.V(0).Infof("re-queued %d recovered pending deletions", len(ids))
	}()
	return true
}