mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-12 23:09:33 +01:00
Chris LuandClaude Opus 5.5
e0f8438284
* mount: keep the filer status in the flush create error flushMetadataToFiler formatted the CreateEntry error with %v, so callers classifying it saw only the message, which includes the file path. Wrap it so the gRPC status the filer sent stays reachable. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: retry a writeback metadata flush until the filer answers With -writebackCache, close() returns before the chunk references reach the filer. The deferred flush gave up after four attempts, about a minute once each attempt's own filer retries are counted, then released the handle: a longer filer outage dropped the file even though write() and close() had succeeded, and its uploaded chunks became orphans. Keep retrying for as long as the filer is unreachable, as the writeback chunk uploads already do. A filer that answers with an error still ends the flush after the bounded attempts, and an unlink of the draining file still ends it on the next attempt. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: retry only an unanswered filer call without a bound The unbounded phase used util.IsTransientError, which falls back to matching the error text. A refusal the filer sends in its response (read-only, name too long, a plain error string) has no gRPC status, and the flush error carries the file path, so a file named like "unavailable-transport.log" kept retrying against a filer that answered. Retry without a bound only when the call failed with a gRPC Unavailable status: the dial, the connection or the mount's call deadline gave out. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
76 lines
2.2 KiB
Go
76 lines
2.2 KiB
Go
package mount
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
const metadataFlushRetries = 3
|
|
|
|
// metadataFlushSleep waits for d or until ctx is cancelled. Overridable in tests.
|
|
var metadataFlushSleep = func(ctx context.Context, d time.Duration) {
|
|
timer := time.NewTimer(d)
|
|
defer timer.Stop()
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
|
|
func retryMetadataFlush(ctx context.Context, flush func() error, onRetry func(nextAttempt, totalAttempts int, backoff time.Duration, err error)) error {
|
|
return retryMetadataFlushIf(ctx, flush, nil, onRetry)
|
|
}
|
|
|
|
// retryMetadataFlushUntil runs retryMetadataFlush, then keeps retrying every
|
|
// util.RetryWaitTime for as long as keepRetrying accepts the error. A zero
|
|
// totalAttempts in onRetry marks those unbounded attempts.
|
|
func retryMetadataFlushUntil(ctx context.Context, flush func() error, keepRetrying func(error) bool, onRetry func(nextAttempt, totalAttempts int, backoff time.Duration, err error)) error {
|
|
err := retryMetadataFlush(ctx, flush, onRetry)
|
|
for attempt := metadataFlushRetries + 2; err != nil && keepRetrying(err) && ctx.Err() == nil; attempt++ {
|
|
if onRetry != nil {
|
|
onRetry(attempt, 0, util.RetryWaitTime, err)
|
|
}
|
|
metadataFlushSleep(ctx, util.RetryWaitTime)
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
err = flush()
|
|
}
|
|
return err
|
|
}
|
|
|
|
// retryMetadataFlushIf retries flush with exponential backoff, stopping early
|
|
// when shouldRetry returns false (clearly permanent errors) or when ctx is
|
|
// cancelled (the FUSE request was interrupted, e.g. the process was killed).
|
|
func retryMetadataFlushIf(ctx context.Context, flush func() error, shouldRetry func(error) bool, onRetry func(nextAttempt, totalAttempts int, backoff time.Duration, err error)) error {
|
|
totalAttempts := metadataFlushRetries + 1
|
|
var err error
|
|
for attempt := 1; attempt <= totalAttempts; attempt++ {
|
|
err = flush()
|
|
if err == nil {
|
|
break
|
|
}
|
|
if attempt == totalAttempts {
|
|
break
|
|
}
|
|
if shouldRetry != nil && !shouldRetry(err) {
|
|
break
|
|
}
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
|
|
backoff := time.Duration(1<<uint(attempt-1)) * time.Second
|
|
if onRetry != nil {
|
|
onRetry(attempt+1, totalAttempts, backoff, err)
|
|
}
|
|
metadataFlushSleep(ctx, backoff)
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
}
|
|
return err
|
|
}
|