seaweedfs/weed/util/retry.go

package util

import (
	"context"
	"errors"
	"io"
	"math/rand/v2"
	"net"
	"strings"
	"syscall"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"google.golang.org/genproto/googleapis/rpc/errdetails"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

var RetryWaitTime = 6 * time.Second

// ErrLockContended marks a lock acquisition abandoned instead of waited on —
// the caller already holds other locks and cannot queue without inverting
// their order. The holder's write settles the contested state, so callers
// treat it as transient and retry.
var ErrLockContended = errors.New("a local write holds the entry lock")

// transientErrorMessages are substrings of failures that a later attempt is
// likely to get past: connection resets, timeouts, and the throttling or
// overload replies S3 and gRPC hand back. Cloud SDKs bury the underlying net
// error in an opaque wrapper with no Unwrap, so the message is often all
// that is left to match on. Compared case-insensitively, so entries are lower
// case: the same condition reaches different layers capitalized differently
// (a volume server relays its idle timeout as "I/O timeout" inside JSON).
var transientErrorMessages = []string{
	"transport",
	"connection reset",
	"connection refused",
	"broken pipe",
	"unexpected eof",
	"i/o timeout",
	"tls handshake timeout",
	"no such host",
	"no route to host",
	"network is unreachable",
	"client.timeout",
	"requesterror",
	"requesttimeout",
	"slowdown",
	"throttling",
	"ratelimitexceeded",
	"rate limit",
	"too many requests",
	"internalerror",
	"resourceexhausted",
	"unavailable",
}

// IsTransientErrorMessage reports whether an error message describes a network
// or service condition worth retrying. Callers holding an error should use
// IsTransientError; this is for paths that only carry the text, such as the
// per-file status strings in a batch delete response.
func IsTransientErrorMessage(msg string) bool {
	lower := strings.ToLower(msg)
	for _, transient := range transientErrorMessages {
		if strings.Contains(lower, transient) {
			return true
		}
	}
	return false
}

// ServerStatus returns the status a gRPC server sent, when the error chain
// still carries one. Unlike status.FromError it keeps the server's own message
// instead of the whole wrapped string: callers routinely format a path the
// client chose into their wrapper, so a classifier that matches substrings has
// to read what the server said and not what the caller added around it.
func ServerStatus(err error) (*status.Status, bool) {
	var carrier interface{ GRPCStatus() *status.Status }
	if !errors.As(err, &carrier) {
		return nil, false
	}
	st := carrier.GRPCStatus()
	return st, st != nil
}

// IsTransientError reports whether err is a network or service condition worth
// retrying. A cancelled or expired context never is: the caller is already gone.
func IsTransientError(err error) bool {
	if err == nil {
		return false
	}
	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
		return false
	}
	if errors.Is(err, ErrLockContended) {
		return true
	}
	if errors.Is(err, io.ErrUnexpectedEOF) ||
		errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.ECONNABORTED) ||
		errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.EPIPE) ||
		errors.Is(err, syscall.ETIMEDOUT) {
		return true
	}
	var netErr net.Error
	if errors.As(err, &netErr) && netErr.Timeout() {
		return true
	}
	if st, ok := ServerStatus(err); ok {
		if st.Code() == codes.Canceled {
			return strings.Contains(st.Message(), "the client connection is closing")
		}
		return st.Code() == codes.Unavailable || st.Code() == codes.ResourceExhausted ||
			IsTransientErrorMessage(st.Message())
	}
	return IsTransientErrorMessage(err.Error())
}

func Retry(name string, job func() error) (err error) {
	waitTime := time.Second
	hasErr := false
	for waitTime < RetryWaitTime {
		err = job()
		if err == nil {
			if hasErr {
				glog.V(0).Infof("retry %s successfully", name)
			}
			waitTime = time.Second
			break
		}
		if IsTransientError(err) {
			hasErr = true
			glog.V(0).Infof("retry %s: err: %v", name, err)
		} else {
			break
		}
		time.Sleep(waitTime)
		waitTime += waitTime / 2
	}
	return err
}

func MultiRetry(name string, errList []string, job func() error) (err error) {
	waitTime := time.Second
	hasErr := false
	for waitTime < RetryWaitTime {
		err = job()
		if err == nil {
			if hasErr {
				glog.V(0).Infof("retry %s successfully", name)
			}
			waitTime = time.Second
			break
		}
		if containErr(err.Error(), errList) {
			hasErr = true
			glog.V(0).Infof("retry %s: err: %v", name, err)
		} else {
			break
		}
		time.Sleep(waitTime)
		waitTime += waitTime / 2
	}
	return err
}

// RetryOnError retries job with the same bounded backoff as MultiRetry, but
// decides retriability with a predicate instead of an error-substring list.
func RetryOnError(name string, shouldRetry func(error) bool, job func() error) (err error) {
	waitTime := time.Second
	hasErr := false
	for waitTime < RetryWaitTime {
		err = job()
		if err == nil {
			if hasErr {
				glog.V(0).Infof("retry %s successfully", name)
			}
			break
		}
		if shouldRetry(err) {
			hasErr = true
			glog.V(0).Infof("retry %s: err: %v", name, err)
		} else {
			break
		}
		time.Sleep(waitTime)
		waitTime += waitTime / 2
	}
	return err
}

// RetryUntil retries until the job returns no error or onErrFn returns false
func RetryUntil(name string, job func() error, onErrFn func(err error) (shouldContinue bool)) error {
	waitTime := time.Second
	for {
		err := job()
		if err == nil {
			waitTime = time.Second
			return nil
		}
		if onErrFn(err) {
			if strings.Contains(err.Error(), "transport") || strings.Contains(err.Error(), "ResourceExhausted") || strings.Contains(err.Error(), "Unavailable") {
				glog.V(0).Infof("retry %s: err: %v", name, err)
			}
			time.Sleep(waitTime)
			if waitTime < RetryWaitTime {
				waitTime += waitTime / 2
			}
			continue
		} else {
			return err
		}
	}
}

// RetryWithBackoff retries an operation on codes.Unavailable errors with exponential
// backoff, respecting context cancellation and a maximum retry duration.
// Returns nil on success, ctx.Err() on context cancellation, or the last error
// when maxDuration is exceeded or a non-retriable error occurs.
func RetryWithBackoff(ctx context.Context, name string, maxDuration time.Duration, shouldRetry func(error) bool, operation func() error) error {
	waitTime := time.Second
	maxWaitTime := RetryWaitTime
	deadline := time.Now().Add(maxDuration)
	var lastErr error
	for {
		if ctx.Err() != nil {
			return ctx.Err()
		}
		if time.Until(deadline) <= 0 {
			if lastErr != nil {
				glog.V(0).Infof("retry %s: giving up after %v: %v", name, maxDuration, lastErr)
				return lastErr
			}
		}
		err := operation()
		if err == nil {
			return nil
		}
		lastErr = err
		if !shouldRetry(err) {
			return err
		}
		remaining := time.Until(deadline)
		if remaining <= 0 {
			glog.V(0).Infof("retry %s: giving up after %v: %v", name, maxDuration, err)
			return err
		}
		sleepTime := waitTime
		if sleepTime > maxWaitTime {
			sleepTime = maxWaitTime
		}
		sleepTime = shortenStep(sleepTime, err, retryHintSlack())
		if sleepTime > remaining {
			sleepTime = remaining
		}
		glog.V(1).Infof("retry %s: retrying in %v: %v", name, sleepTime, err)
		timer := retryBackoffTimer(sleepTime)
		select {
		case <-ctx.Done():
			if !timer.Stop() {
				<-timer.C
			}
			return ctx.Err()
		case <-timer.C:
		}
		waitTime += waitTime / 2
		if waitTime > maxWaitTime {
			waitTime = maxWaitTime
		}
	}
}

var retryBackoffTimer = time.NewTimer

// retryHintSlack lands a hinted retry just after the server is ready, jittered
// so clients refused together do not retry together.
func retryHintSlack() time.Duration {
	return 50*time.Millisecond + rand.N(250*time.Millisecond+1)
}

// shortenStep returns hint plus slack when that is shorter than step.
// hint < step-slack, not hint+slack < step, so a huge hint cannot overflow.
func shortenStep(step time.Duration, err error, slack time.Duration) time.Duration {
	if hint, ok := serverRetryDelay(err); ok && hint < step-slack {
		return hint + slack
	}
	return step
}

// serverRetryDelay returns a valid, positive gRPC RetryInfo delay; anything
// else would let a misbehaving server turn the backoff into tight polling.
func serverRetryDelay(err error) (time.Duration, bool) {
	st, ok := ServerStatus(err)
	if !ok {
		return 0, false
	}
	for _, d := range st.Details() {
		if ri, ok := d.(*errdetails.RetryInfo); ok && ri.GetRetryDelay().CheckValid() == nil {
			if delay := ri.GetRetryDelay().AsDuration(); delay > 0 {
				return delay, true
			}
		}
	}
	return 0, false
}

// Nvl return the first non-empty string
func Nvl(values ...string) string {
	for _, s := range values {
		if s != "" {
			return s
		}
	}
	return ""
}

func containErr(err string, errList []string) bool {
	for _, e := range errList {
		if strings.Contains(err, e) {
			return true
		}
	}
	return false
}