package mount import ( "context" "time" "github.com/seaweedfs/seaweedfs/weed/util" ) const metadataFlushRetries = 3 // metadataFlushSleep waits for d or until ctx is cancelled. Overridable in tests. var metadataFlushSleep = func(ctx context.Context, d time.Duration) { timer := time.NewTimer(d) defer timer.Stop() select { case <-ctx.Done(): case <-timer.C: } } func retryMetadataFlush(ctx context.Context, flush func() error, onRetry func(nextAttempt, totalAttempts int, backoff time.Duration, err error)) error { return retryMetadataFlushIf(ctx, flush, nil, onRetry) } // retryMetadataFlushUntil runs retryMetadataFlush, then keeps retrying every // util.RetryWaitTime for as long as keepRetrying accepts the error. A zero // totalAttempts in onRetry marks those unbounded attempts. func retryMetadataFlushUntil(ctx context.Context, flush func() error, keepRetrying func(error) bool, onRetry func(nextAttempt, totalAttempts int, backoff time.Duration, err error)) error { err := retryMetadataFlush(ctx, flush, onRetry) for attempt := metadataFlushRetries + 2; err != nil && keepRetrying(err) && ctx.Err() == nil; attempt++ { if onRetry != nil { onRetry(attempt, 0, util.RetryWaitTime, err) } metadataFlushSleep(ctx, util.RetryWaitTime) if ctx.Err() != nil { break } err = flush() } return err } // retryMetadataFlushIf retries flush with exponential backoff, stopping early // when shouldRetry returns false (clearly permanent errors) or when ctx is // cancelled (the FUSE request was interrupted, e.g. the process was killed). func retryMetadataFlushIf(ctx context.Context, flush func() error, shouldRetry func(error) bool, onRetry func(nextAttempt, totalAttempts int, backoff time.Duration, err error)) error { totalAttempts := metadataFlushRetries + 1 var err error for attempt := 1; attempt <= totalAttempts; attempt++ { err = flush() if err == nil { break } if attempt == totalAttempts { break } if shouldRetry != nil && !shouldRetry(err) { break } if ctx.Err() != nil { break } backoff := time.Duration(1<