seaweedfs/weed/filer/filer_deletion_persist_test.go

package filer

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

// readPersistedLedger pulls the persisted ledger payload straight from the store
// the filer is wired to, the same way a restarting filer would.
func readPersistedLedger(t *testing.T, f *Filer) ([]string, bool) {
	t.Helper()
	raw, err := f.Store.KvGet(context.Background(), []byte(f.deletionLedgerKey()))
	if err != nil {
		if err == ErrKvNotFound {
			return nil, false
		}
		t.Fatalf("unexpected error reading ledger: %v", err)
	}
	var ids []string
	if err := json.Unmarshal(raw, &ids); err != nil {
		t.Fatalf("persisted payload not valid JSON: %v", err)
	}
	return ids, true
}

// withPersistedConfig pins the ledger knobs at viper's override tier so the test
// sees deterministic semantics regardless of global SetDefault ordering (the
// production getters SetDefault(true), which would otherwise win the tier race).
// The overrides are restored on cleanup so later tests see production defaults.
func withPersistedConfig(t *testing.T, enabled bool) {
	t.Helper()
	v := util.GetViper()
	v.Set("filer.deleteQueue.persist", enabled)
	v.Set("filer.deleteQueue.recoveryGrace", defaultDeletionRecoveryGrace)
	t.Cleanup(func() {
		v.Set("filer.deleteQueue.persist", true)
		v.Set("filer.deleteQueue.recoveryGrace", defaultDeletionRecoveryGrace)
	})
}

// newLedgerTestFiler builds a Filer with the pieces the ledger touches, backed by
// a real (stub) store, bypassing NewFiler's master/aggregator wiring.
func newLedgerTestFiler(store FilerStore) *Filer {
	return &Filer{
		FileIdDeletionQueue: util.NewUnboundedQueue(),
		DeletionRetryQueue:  NewDeletionRetryQueue(),
		deletionQuit:        make(chan struct{}),
		Store:               NewFilerStoreWrapper(store),
	}
}

// TestDeletionLedgerSnapshotAndRecover is the core guarantee: enqueued-but-
// unconfirmed deletions survive a simulated process restart and land back in the
// hot queue (minus whatever was confirmed terminal in the meantime).
func TestDeletionLedgerSnapshotAndRecover(t *testing.T) {
	withPersistedConfig(t, true)
	store := newStubFilerStore()
	f := newLedgerTestFiler(store)

	f.queueDeletions("1,01", "1,02", "1,03")
	f.snapshotDeletionLedger()

	persisted, ok := readPersistedLedger(t, f)
	if !ok {
		t.Fatalf("expected persisted ledger under %q", KvKeyDeletionLedger)
	}
	if len(persisted) != 3 {
		t.Fatalf("expected 3 persisted ids, got %d: %v", len(persisted), persisted)
	}
	for _, want := range []string{"1,01", "1,02", "1,03"} {
		if !contains(persisted, want) {
			t.Errorf("expected %q in persisted ledger, got %v", want, persisted)
		}
	}

	// Confirmed-gone for one id, then snapshot again.
	f.forgetDeletion("1,02")
	f.snapshotDeletionLedger()
	persisted, _ = readPersistedLedger(t, f)
	if len(persisted) != 2 {
		t.Fatalf("expected 2 persisted ids after forget, got %d: %v", len(persisted), persisted)
	}
	if contains(persisted, "1,02") {
		t.Errorf("forgotten id still in persisted ledger: %v", persisted)
	}

	// --- simulated restart: a fresh Filer with NO ledger in memory must
	// recover the still-pending ids from the store and re-queue them. ---
	f2 := newLedgerTestFiler(store)
	// recoveryGrace 0 so the background re-enqueue is near-instant.
	util.GetViper().Set("filer.deleteQueue.recoveryGrace", time.Duration(0))
	f2.reloadDeletionLedger()

	// Wait for the async recovery goroutine to drain into the hot queue.
	seen := make(map[string]bool)
	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		f2.FileIdDeletionQueue.Consume(func(ids []string) {
			for _, id := range ids {
				seen[id] = true
			}
		})
		if len(seen) >= 2 {
			break
		}
		time.Sleep(10 * time.Millisecond)
	}

	if !seen["1,01"] || !seen["1,03"] {
		t.Fatalf("expected recovered ids 1,01 and 1,03 back in hot queue, got %v", seen)
	}
	if seen["1,02"] {
		t.Errorf("forgotten id 1,02 must NOT come back: %v", seen)
	}
}

// TestDeletionLedgerRetryKeepsEntry pins the key semantic: a non-terminal
// (retryable) failure must keep the entry persisted, so a crash mid-retry does
// not orphan the chunk. This is the whole reason forgetDeletion is only called on
// success / not-found / permanent.
func TestDeletionLedgerRetryKeepsEntry(t *testing.T) {
	withPersistedConfig(t, true)
	store := newStubFilerStore()
	f := newLedgerTestFiler(store)

	f.queueDeletions("9,01")
	f.snapshotDeletionLedger()

	// A retryable outcome deliberately does NOT call forgetDeletion, so the id
	// must still be in the durable ledger.
	persisted, ok := readPersistedLedger(t, f)
	if !ok {
		t.Fatalf("ledger missing — retryable ids must remain persisted")
	}
	if len(persisted) != 1 || persisted[0] != "9,01" {
		t.Fatalf("retryable id must stay in ledger, got %v", persisted)
	}
}

// TestDeletionLedgerDisabled verifies the kill switch: with persist off, nothing
// reaches KV, and reload refuses to recover even if a ledger exists in the store
// (so an operator turning the feature off never resurrects a stale ledger).
func TestDeletionLedgerDisabled(t *testing.T) {
	// First, with persistence ON, put a real ledger into the store.
	withPersistedConfig(t, true)
	store := newStubFilerStore()
	fOn := newLedgerTestFiler(store)
	fOn.queueDeletions("5,01")
	fOn.snapshotDeletionLedger()
	if _, ok := readPersistedLedger(t, fOn); !ok {
		t.Fatalf("setup: expected a persisted ledger with persist enabled")
	}

	// Now flip the switch OFF and check the contract.
	withPersistedConfig(t, false)

	// 1. Snapshot does not touch KV (no new writes, no clear).
	f := newLedgerTestFiler(store)
	f.queueDeletions("6,01") // in-memory only
	f.snapshotDeletionLedger()
	persisted, ok := readPersistedLedger(t, f)
	if !ok {
		t.Fatalf("setup: ledger should still exist from the enabled phase")
	}
	// The enabled-phase ledger had "5,01"; disabled phase must not have added "6,01".
	if contains(persisted, "6,01") {
		t.Errorf("persist disabled but snapshot wrote 6,01: %v", persisted)
	}

	// 2. Reload with the switch off must NOT recover, even though a ledger exists.
	f2 := newLedgerTestFiler(store)
	f2.reloadDeletionLedger()
	if f2.pendingDeletionCount() != 0 {
		t.Fatalf("reload must be a no-op when disabled, got %d pending", f2.pendingDeletionCount())
	}
}

// Filers sharing one metadata store must not overwrite each other's ledgers:
// each filer keys its ledger by its own address.
func TestDeletionLedgerScopedPerFiler(t *testing.T) {
	withPersistedConfig(t, true)
	store := newStubFilerStore()
	fA := newLedgerTestFiler(store)
	fA.Dlm = lock_manager.NewDistributedLockManager("filer-a:8888")
	fB := newLedgerTestFiler(store)
	fB.Dlm = lock_manager.NewDistributedLockManager("filer-b:8888")

	fA.queueDeletions("1,01")
	fB.queueDeletions("2,02")
	fA.snapshotDeletionLedger()
	fB.snapshotDeletionLedger()

	idsA, okA := readPersistedLedger(t, fA)
	idsB, okB := readPersistedLedger(t, fB)
	if !okA || !okB {
		t.Fatalf("both filers must have their own ledger, got %v %v", idsA, idsB)
	}
	if !contains(idsA, "1,01") || contains(idsA, "2,02") {
		t.Fatalf("filer A ledger wrong: %v", idsA)
	}
	if !contains(idsB, "2,02") || contains(idsB, "1,01") {
		t.Fatalf("filer B ledger wrong: %v", idsB)
	}

	// A restarted filer on B's address recovers only B's pending deletions.
	fB2 := newLedgerTestFiler(store)
	fB2.Dlm = lock_manager.NewDistributedLockManager("filer-b:8888")
	if !fB2.reloadDeletionLedger() {
		t.Fatalf("reload should succeed")
	}
	if fB2.pendingDeletionCount() != 1 {
		t.Fatalf("B's restart should recover exactly its own id, got %d", fB2.pendingDeletionCount())
	}
}

// A ledger bigger than one store value must still persist: it splits into part
// keys under a manifest, and shrinking below the part size removes the parts.
func TestDeletionLedgerChunked(t *testing.T) {
	withPersistedConfig(t, true)
	store := newStubFilerStore()
	f := newLedgerTestFiler(store)

	var ids []string
	for i := 0; i < 4000; i++ {
		ids = append(ids, fmt.Sprintf("7,%08xbeefcafe", i))
	}
	f.queueDeletions(ids...)
	f.snapshotDeletionLedger()

	raw, err := f.Store.KvGet(context.Background(), []byte(f.deletionLedgerKey()))
	if err != nil || len(raw) == 0 || raw[0] != '{' {
		t.Fatalf("expected a chunked manifest, got %v %q", err, raw)
	}

	f2 := newLedgerTestFiler(store)
	if !f2.reloadDeletionLedger() {
		t.Fatalf("chunked ledger should reload")
	}
	if got := f2.pendingDeletionCount(); got != len(ids) {
		t.Fatalf("recovered %d ids, want %d", got, len(ids))
	}

	// Forget everything; the next snapshot is a single value again and the
	// part keys are removed.
	for _, id := range ids {
		f2.forgetDeletion(id)
	}
	f2.snapshotDeletionLedger()
	raw, err = f.Store.KvGet(context.Background(), []byte(f2.deletionLedgerKey()))
	if err != nil || len(raw) == 0 || raw[0] != '[' {
		t.Fatalf("expected single-value ledger after shrink, got %v %q", err, raw)
	}
	if _, err := f.Store.KvGet(context.Background(), deletionLedgerPartKey(f2.deletionLedgerKey(), 0)); err != ErrKvNotFound {
		t.Fatalf("stale part key must be removed, got %v", err)
	}
}

// A startup ledger read that fails for a reason other than not-found must not
// let snapshots overwrite the unread ledger with a partial set.
func TestDeletionLedgerReadFailureBlocksPersistence(t *testing.T) {
	withPersistedConfig(t, true)
	store := newStubFilerStore()
	f := newLedgerTestFiler(store)
	store.kvGetErr = errors.New("kv read down")

	if f.reloadDeletionLedger() {
		t.Fatalf("reload should report the ledger as unusable")
	}

	f.queueDeletions("1,01")
	f.snapshotDeletionLedger()
	if len(store.kv) != 0 {
		t.Fatalf("snapshot must not overwrite a ledger that was never read, wrote %v", store.kv)
	}

	// The block is not permanent: once the store reads again, persistence
	// resumes — a transient failure must not disable the ledger for the life
	// of the process.
	store.kvGetErr = nil
	f.snapshotDeletionLedger()
	persisted, ok := readPersistedLedger(t, f)
	if !ok || len(persisted) != 1 || persisted[0] != "1,01" {
		t.Fatalf("persistence should resume once the ledger reads, got %v", persisted)
	}
}

// Recovered ids join the pending set immediately — before the grace delay — so
// an early snapshot rewrites the recovered ids instead of dropping them.
func TestDeletionLedgerRecoveryKeepsIdsPending(t *testing.T) {
	withPersistedConfig(t, true)
	store := newStubFilerStore()
	f := newLedgerTestFiler(store)
	f.queueDeletions("1,01", "1,02")
	f.snapshotDeletionLedger()

	util.GetViper().Set("filer.deleteQueue.recoveryGrace", time.Hour)
	f2 := newLedgerTestFiler(store)
	if !f2.reloadDeletionLedger() {
		t.Fatalf("reload should succeed")
	}
	// Still inside the grace window, but the ids are already pending.
	if f2.pendingDeletionCount() != 2 {
		t.Fatalf("recovered ids must be pending immediately, got %d", f2.pendingDeletionCount())
	}
	// And an early shutdown snapshot keeps them.
	f2.snapshotDeletionLedger()
	persisted, ok := readPersistedLedger(t, f2)
	if !ok || len(persisted) != 2 {
		t.Fatalf("snapshot during grace must carry recovered ids, got %v", persisted)
	}
}

// TestDeletionLedgerZeroValueFiler proves nil-map safety for the struct literals
// used across the test suite (they never call NewFiler).
func TestDeletionLedgerZeroValueFiler(t *testing.T) {
	withPersistedConfig(t, true)
	f := &Filer{
		FileIdDeletionQueue: util.NewUnboundedQueue(),
		DeletionRetryQueue:  NewDeletionRetryQueue(),
		deletionQuit:        make(chan struct{}),
	}
	f.queueDeletions("0,01")
	if f.pendingDeletionCount() != 1 {
		t.Fatalf("zero-value filer should lazily init the ledger, got %d", f.pendingDeletionCount())
	}
	f.forgetDeletion("0,01")
	if f.pendingDeletionCount() != 0 {
		t.Fatalf("forget on zero-value filer failed, got %d", f.pendingDeletionCount())
	}
}

// A manifest that references a part the store cannot return is corruption, not
// absence: reload must report failure and nothing may overwrite the surviving
// manifest with only the in-memory set.
func TestDeletionLedgerMissingPartBlocks(t *testing.T) {
	withPersistedConfig(t, true)
	store := newStubFilerStore()
	f := newLedgerTestFiler(store)

	key := f.deletionLedgerKey()
	store.kv[key] = []byte(`{"parts":2,"gen":4}`)
	store.kv[string(deletionLedgerGenPartKey(key, 4, 0))] = []byte(`["1,01"]`)
	// Part 1 of generation 4 is missing.

	if f.reloadDeletionLedger() {
		t.Fatalf("a manifest referencing a missing part is corrupt, not absent")
	}
	f.queueDeletions("9,99")
	f.snapshotDeletionLedger()
	if got := string(store.kv[key]); got != `{"parts":2,"gen":4}` {
		t.Fatalf("unread ledger must not be overwritten, got %q", got)
	}
}

// A filer restarting under a new advertised address must still find its
// previous ledger: the index lists scoped keys, the ids are published under
// the new key first, and only then is the stranded key removed.
func TestDeletionLedgerAddressChangeClaim(t *testing.T) {
	withPersistedConfig(t, true)
	store := newStubFilerStore()
	fOld := newLedgerTestFiler(store)
	fOld.Dlm = lock_manager.NewDistributedLockManager("filer-old:8888")
	fOld.queueDeletions("1,01", "1,02")
	fOld.snapshotDeletionLedger()

	fNew := newLedgerTestFiler(store)
	fNew.Dlm = lock_manager.NewDistributedLockManager("filer-new:9999")
	if !fNew.reloadDeletionLedger() {
		t.Fatalf("reload should claim the stranded ledger")
	}
	if fNew.pendingDeletionCount() != 2 {
		t.Fatalf("expected 2 claimed ids, got %d", fNew.pendingDeletionCount())
	}
	if _, err := store.KvGet(context.Background(), []byte(fOld.deletionLedgerKey())); err != ErrKvNotFound {
		t.Fatalf("stranded ledger must be removed after claim, got %v", err)
	}
	persisted, ok := readPersistedLedger(t, fNew)
	if !ok || len(persisted) != 2 {
		t.Fatalf("claimed ids must be durably stored under the new key, got %v", persisted)
	}
	indexKeys, _ := fNew.readLedgerIndex(context.Background())
	if contains(indexKeys, fOld.deletionLedgerKey()) {
		t.Fatalf("claimed key must leave the index, got %v", indexKeys)
	}
}

// Every multipart snapshot writes a fresh generation's part keys and publishes
// the manifest only after all parts land, so the committed generation is never
// overwritten. The previous generation is removed after publication.
func TestDeletionLedgerGenerationPublish(t *testing.T) {
	withPersistedConfig(t, true)
	store := newStubFilerStore()
	f := newLedgerTestFiler(store)

	var ids []string
	for i := 0; i < 4000; i++ {
		ids = append(ids, fmt.Sprintf("7,%08xbeefcafe", i))
	}
	f.queueDeletions(ids...)
	f.snapshotDeletionLedger()
	key := f.deletionLedgerKey()
	if _, err := store.KvGet(context.Background(), deletionLedgerGenPartKey(key, 1, 0)); err != nil {
		t.Fatalf("generation 1 part must exist, got %v", err)
	}

	f.queueDeletions("8,08")
	f.snapshotDeletionLedger()
	if _, err := store.KvGet(context.Background(), deletionLedgerGenPartKey(key, 2, 0)); err != nil {
		t.Fatalf("generation 2 part must exist, got %v", err)
	}
	if _, err := store.KvGet(context.Background(), deletionLedgerGenPartKey(key, 1, 0)); err != ErrKvNotFound {
		t.Fatalf("generation 1 part must be cleaned after publication, got %v", err)
	}
	var manifest struct {
		Parts int `json:"parts"`
		Gen   int `json:"gen"`
	}
	raw, _ := store.KvGet(context.Background(), []byte(key))
	if err := json.Unmarshal(raw, &manifest); err != nil || manifest.Gen != 2 {
		t.Fatalf("manifest must publish generation 2, got %q", raw)
	}
}

// A failed cleanup of a superseded generation is retried by the next snapshot
// instead of leaking the part keys.
func TestDeletionLedgerStalePartsRetried(t *testing.T) {
	withPersistedConfig(t, true)
	store := newStubFilerStore()
	f := newLedgerTestFiler(store)

	var ids []string
	for i := 0; i < 4000; i++ {
		ids = append(ids, fmt.Sprintf("7,%08xbeefcafe", i))
	}
	f.queueDeletions(ids...)
	f.snapshotDeletionLedger()
	key := f.deletionLedgerKey()

	store.kvDeleteErr = errors.New("delete down")
	f.queueDeletions("8,08")
	f.snapshotDeletionLedger()
	if len(f.deletionLedgerStale) == 0 {
		t.Fatalf("failed part deletes must be tracked for retry")
	}
	store.kvDeleteErr = nil
	f.queueDeletions("8,09")
	f.snapshotDeletionLedger()
	if len(f.deletionLedgerStale) != 0 {
		t.Fatalf("stale parts must flush once deletes work, got %v", f.deletionLedgerStale)
	}
	if _, err := store.KvGet(context.Background(), deletionLedgerGenPartKey(key, 1, 0)); err != ErrKvNotFound {
		t.Fatalf("stale generation part must be deleted, got %v", err)
	}
}

// A pre-scoping ledger under the base key must migrate only after the scoped
// copy is durable: the recovered ids are written under the scoped key first,
// then the base key is removed.
func TestDeletionLedgerLegacyMigrationDurable(t *testing.T) {
	withPersistedConfig(t, true)
	store := newStubFilerStore()
	store.kv[KvKeyDeletionLedger] = []byte(`["1,01","1,02"]`)

	f := newLedgerTestFiler(store)
	f.Dlm = lock_manager.NewDistributedLockManager("filer-new:9999")
	if !f.reloadDeletionLedger() {
		t.Fatalf("reload should migrate the legacy ledger")
	}
	persisted, ok := readPersistedLedger(t, f)
	if !ok || len(persisted) != 2 {
		t.Fatalf("legacy ids must land under the scoped key first, got %v", persisted)
	}
	if _, err := store.KvGet(context.Background(), []byte(KvKeyDeletionLedger)); err != ErrKvNotFound {
		t.Fatalf("legacy key must be removed once the scoped write is durable, got %v", err)
	}
	if f.pendingDeletionCount() != 2 {
		t.Fatalf("migrated ids must join the pending set, got %d", f.pendingDeletionCount())
	}
}

// An indexed ledger that cannot be read must not be skipped: claiming the
// readable ones and leaving the unreadable one behind would strand it, since
// the index is only searched while the filer's own key is absent. The claim
// aborts so the reload is retried.
func TestDeletionLedgerUnreadableForeignBlocks(t *testing.T) {
	withPersistedConfig(t, true)
	store := newStubFilerStore()
	stranded := KvKeyDeletionLedger + ".filer-old:8888"
	store.kv[stranded] = []byte(`{"parts":2,"gen":1}`) // manifest without parts = corrupt
	index, _ := json.Marshal([]string{stranded})
	store.kv[KvKeyDeletionLedgerIndex] = index

	f := newLedgerTestFiler(store)
	f.Dlm = lock_manager.NewDistributedLockManager("filer-new:9999")
	if f.reloadDeletionLedger() {
		t.Fatalf("an unreadable indexed ledger must fail the reload")
	}
	if _, err := store.KvGet(context.Background(), []byte(stranded)); err != nil {
		t.Fatalf("unreadable ledger must be left untouched, got %v", err)
	}
	if _, ok := readPersistedLedger(t, f); ok {
		t.Fatalf("no scoped ledger may be written while a claim source is unreadable")
	}
}

// A live filer's ledger is not deleted when it republished between the claim's
// read and cleanup: the source stays and its newer ids merge into the claim.
// The hook swaps the source content on its second read — the claim's first
// read sees the old set, the verification read sees the republished one,
// exactly like a peer snapshot landing mid-claim.
func TestDeletionLedgerClaimKeepsChangedSource(t *testing.T) {
	withPersistedConfig(t, true)
	store := newStubFilerStore()
	srcKey := KvKeyDeletionLedger + ".filer-old:8888"
	store.kv[srcKey] = []byte(`["1,01","1,02"]`)
	index, _ := json.Marshal([]string{srcKey})
	store.kv[KvKeyDeletionLedgerIndex] = index

	srcReads := 0
	store.kvGetHook = func(key []byte) {
		if string(key) == srcKey {
			srcReads++
			if srcReads == 2 {
				store.kv[srcKey] = []byte(`["1,01","1,02","1,03"]`)
			}
		}
	}

	f := newLedgerTestFiler(store)
	f.Dlm = lock_manager.NewDistributedLockManager("filer-new:9999")
	if !f.reloadDeletionLedger() {
		t.Fatalf("claim should succeed")
	}
	if got := string(store.kv[srcKey]); got != `["1,01","1,02","1,03"]` {
		t.Fatalf("republished source must be left intact, got %q", got)
	}
	if f.pendingDeletionCount() != 3 {
		t.Fatalf("republished ids must merge into the pending set, got %d", f.pendingDeletionCount())
	}
	persisted, ok := readPersistedLedger(t, f)
	if !ok || len(persisted) != 3 {
		t.Fatalf("merged ids must be durable under our key, got %v", persisted)
	}
}

// An expired retry item must not erase a newer enqueue for the same file id:
// expiry forgets only the epoch the retry item recorded.
func TestForgetDeletionEpochSkipsNewer(t *testing.T) {
	withPersistedConfig(t, true)
	f := newLedgerTestFiler(newStubFilerStore())

	f.queueDeletions("1,01")
	oldEpoch := f.deletionEpoch("1,01")
	f.queueDeletions("1,01") // re-enqueue bumps the epoch
	newEpoch := f.deletionEpoch("1,01")
	if oldEpoch == newEpoch {
		t.Fatalf("re-enqueue must bump the epoch")
	}

	f.forgetDeletionEpoch("1,01", oldEpoch)
	if f.pendingDeletionCount() != 1 {
		t.Fatalf("stale expiry must not erase a newer enqueue")
	}
	f.forgetDeletionEpoch("1,01", newEpoch)
	if f.pendingDeletionCount() != 0 {
		t.Fatalf("matching expiry must forget the id")
	}
}

func contains(haystack []string, needle string) bool {
	for _, s := range haystack {
		if s == needle {
			return true
		}
	}
	return false
}