seaweedfs/weed/mount/metadata_flush_retry_test.go

package mount

import (
	"context"
	"errors"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/util"
)

func TestRetryMetadataFlushEventuallySucceeds(t *testing.T) {
	originalSleep := metadataFlushSleep
	t.Cleanup(func() {
		metadataFlushSleep = originalSleep
	})

	var sleeps []time.Duration
	metadataFlushSleep = func(ctx context.Context, d time.Duration) {
		sleeps = append(sleeps, d)
	}

	attempts := 0
	err := retryMetadataFlush(context.Background(), func() error {
		attempts++
		if attempts < 3 {
			return errors.New("temporary failure")
		}
		return nil
	}, nil)
	if err != nil {
		t.Fatalf("retryMetadataFlush returned error: %v", err)
	}

	if attempts != 3 {
		t.Fatalf("attempts = %d, want 3", attempts)
	}

	wantSleeps := []time.Duration{time.Second, 2 * time.Second}
	if len(sleeps) != len(wantSleeps) {
		t.Fatalf("sleep count = %d, want %d", len(sleeps), len(wantSleeps))
	}
	for i, want := range wantSleeps {
		if sleeps[i] != want {
			t.Fatalf("sleep[%d] = %v, want %v", i, sleeps[i], want)
		}
	}
}

func TestRetryMetadataFlushReturnsLastError(t *testing.T) {
	originalSleep := metadataFlushSleep
	t.Cleanup(func() {
		metadataFlushSleep = originalSleep
	})

	var sleeps []time.Duration
	metadataFlushSleep = func(ctx context.Context, d time.Duration) {
		sleeps = append(sleeps, d)
	}

	expectedErr := errors.New("permanent failure")
	attempts := 0
	err := retryMetadataFlush(context.Background(), func() error {
		attempts++
		return expectedErr
	}, nil)
	if !errors.Is(err, expectedErr) {
		t.Fatalf("retryMetadataFlush error = %v, want %v", err, expectedErr)
	}

	if attempts != metadataFlushRetries+1 {
		t.Fatalf("attempts = %d, want %d", attempts, metadataFlushRetries+1)
	}

	wantSleeps := []time.Duration{time.Second, 2 * time.Second, 4 * time.Second}
	if len(sleeps) != len(wantSleeps) {
		t.Fatalf("sleep count = %d, want %d", len(sleeps), len(wantSleeps))
	}
	for i, want := range wantSleeps {
		if sleeps[i] != want {
			t.Fatalf("sleep[%d] = %v, want %v", i, sleeps[i], want)
		}
	}
}

// TestRetryMetadataFlushStopsOnCancel verifies that once the flush context is
// done (the flush deadline elapsed against an overwhelmed filer) the retry loop
// abandons its retries immediately instead of sleeping out the backoff, so
// close() is not held longer than the deadline.
func TestRetryMetadataFlushStopsOnCancel(t *testing.T) {
	originalSleep := metadataFlushSleep
	t.Cleanup(func() {
		metadataFlushSleep = originalSleep
	})

	var sleeps []time.Duration
	metadataFlushSleep = func(ctx context.Context, d time.Duration) {
		sleeps = append(sleeps, d)
	}

	ctx, cancel := context.WithCancel(context.Background())
	cancel() // flush deadline already elapsed before the first attempt completes

	attempts := 0
	flushErr := errors.New("filer overwhelmed")
	err := retryMetadataFlush(ctx, func() error {
		attempts++
		return flushErr
	}, nil)
	if !errors.Is(err, flushErr) {
		t.Fatalf("retryMetadataFlush error = %v, want %v", err, flushErr)
	}

	if attempts != 1 {
		t.Fatalf("attempts = %d, want 1 (no retries after cancel)", attempts)
	}
	if len(sleeps) != 0 {
		t.Fatalf("sleeps = %v, want none (no backoff after cancel)", sleeps)
	}
}

func TestRetryMetadataFlushUntilOutlastsOutage(t *testing.T) {
	originalSleep := metadataFlushSleep
	t.Cleanup(func() {
		metadataFlushSleep = originalSleep
	})

	var sleeps []time.Duration
	metadataFlushSleep = func(ctx context.Context, d time.Duration) {
		sleeps = append(sleeps, d)
	}

	const failures = 10
	attempts := 0
	var unbounded []int
	err := retryMetadataFlushUntil(context.Background(), func() error {
		attempts++
		if attempts <= failures {
			return errors.New("filer unreachable")
		}
		return nil
	}, func(error) bool { return true }, func(nextAttempt, totalAttempts int, backoff time.Duration, err error) {
		if totalAttempts == 0 {
			unbounded = append(unbounded, nextAttempt)
		}
	})
	if err != nil {
		t.Fatalf("retryMetadataFlushUntil returned error: %v", err)
	}
	if attempts != failures+1 {
		t.Fatalf("attempts = %d, want %d", attempts, failures+1)
	}

	wantSleeps := []time.Duration{time.Second, 2 * time.Second, 4 * time.Second}
	for len(wantSleeps) < failures {
		wantSleeps = append(wantSleeps, util.RetryWaitTime)
	}
	if len(sleeps) != len(wantSleeps) {
		t.Fatalf("sleeps = %v, want %v", sleeps, wantSleeps)
	}
	for i, want := range wantSleeps {
		if sleeps[i] != want {
			t.Fatalf("sleep[%d] = %v, want %v", i, sleeps[i], want)
		}
	}
	if len(unbounded) != failures-metadataFlushRetries || unbounded[0] != metadataFlushRetries+2 {
		t.Fatalf("unbounded retry attempts = %v, want %d of them starting at %d",
			unbounded, failures-metadataFlushRetries, metadataFlushRetries+2)
	}
}

func TestRetryMetadataFlushUntilStopsOnRejectedError(t *testing.T) {
	originalSleep := metadataFlushSleep
	t.Cleanup(func() {
		metadataFlushSleep = originalSleep
	})
	metadataFlushSleep = func(ctx context.Context, d time.Duration) {}

	transient := errors.New("filer unreachable")
	permanent := errors.New("permission denied")
	attempts := 0
	err := retryMetadataFlushUntil(context.Background(), func() error {
		attempts++
		if attempts < 7 {
			return transient
		}
		return permanent
	}, func(err error) bool { return errors.Is(err, transient) }, nil)
	if !errors.Is(err, permanent) {
		t.Fatalf("retryMetadataFlushUntil error = %v, want %v", err, permanent)
	}
	if attempts != 7 {
		t.Fatalf("attempts = %d, want 7", attempts)
	}

	attempts = 0
	err = retryMetadataFlushUntil(context.Background(), func() error {
		attempts++
		return permanent
	}, func(err error) bool { return errors.Is(err, transient) }, nil)
	if !errors.Is(err, permanent) {
		t.Fatalf("retryMetadataFlushUntil error = %v, want %v", err, permanent)
	}
	if attempts != metadataFlushRetries+1 {
		t.Fatalf("attempts = %d, want the bounded %d", attempts, metadataFlushRetries+1)
	}
}

func TestRetryMetadataFlushUntilStopsOnCancel(t *testing.T) {
	originalSleep := metadataFlushSleep
	t.Cleanup(func() {
		metadataFlushSleep = originalSleep
	})

	ctx, cancel := context.WithCancel(context.Background())
	sleeps := 0
	metadataFlushSleep = func(ctx context.Context, d time.Duration) {
		sleeps++
		if d == util.RetryWaitTime {
			cancel()
		}
	}

	attempts := 0
	err := retryMetadataFlushUntil(ctx, func() error {
		attempts++
		return errors.New("filer unreachable")
	}, func(error) bool { return true }, nil)
	if err == nil {
		t.Fatal("retryMetadataFlushUntil returned nil after cancel")
	}
	if attempts != metadataFlushRetries+1 {
		t.Fatalf("attempts = %d, want %d (none after cancel)", attempts, metadataFlushRetries+1)
	}
	if sleeps != metadataFlushRetries+1 {
		t.Fatalf("sleeps = %d, want %d", sleeps, metadataFlushRetries+1)
	}
}