mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-08 15:41:15 +02:00
Chris Lu
ef109fe9e1
* operation: bound AssignVolume with a deadline AssignVolume ran on context.Background(), so when the filer is overwhelmed the RPC could block indefinitely and wedge every caller holding the connection. Give it a 30s deadline so a stuck assign fails and the caller's retry/error path runs instead of hanging forever. * mount: abort flush when the FUSE request is interrupted On close(), a killed process blocks in fuse_flush waiting for the mount to answer. doFlush ran its metadata CreateEntry on context.Background() and ignored the kernel interrupt channel, so against an overwhelmed filer the flush never completed and the process stayed in uninterruptible sleep -- making the pod un-killable. Derive a context from the FUSE cancel channel in Flush/Fsync and thread it through doFlush -> flushMetadataToFiler -> streamCreateEntry; the retry loop stops as soon as the context is cancelled. Release and the pre-rename flush keep a non-cancellable context since they must finish regardless. * operation: harden the AssignVolume timeout test Make the test double's signal send non-blocking and bound the receive with a timeout so a regression can't wedge the test instead of failing it.
56 lines
1.5 KiB
Go
56 lines
1.5 KiB
Go
package mount
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
)
|
|
|
|
const metadataFlushRetries = 3
|
|
|
|
// metadataFlushSleep waits for d or until ctx is cancelled. Overridable in tests.
|
|
var metadataFlushSleep = func(ctx context.Context, d time.Duration) {
|
|
timer := time.NewTimer(d)
|
|
defer timer.Stop()
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
|
|
func retryMetadataFlush(ctx context.Context, flush func() error, onRetry func(nextAttempt, totalAttempts int, backoff time.Duration, err error)) error {
|
|
return retryMetadataFlushIf(ctx, flush, nil, onRetry)
|
|
}
|
|
|
|
// retryMetadataFlushIf retries flush with exponential backoff, stopping early
|
|
// when shouldRetry returns false (clearly permanent errors) or when ctx is
|
|
// cancelled (the FUSE request was interrupted, e.g. the process was killed).
|
|
func retryMetadataFlushIf(ctx context.Context, flush func() error, shouldRetry func(error) bool, onRetry func(nextAttempt, totalAttempts int, backoff time.Duration, err error)) error {
|
|
totalAttempts := metadataFlushRetries + 1
|
|
var err error
|
|
for attempt := 1; attempt <= totalAttempts; attempt++ {
|
|
err = flush()
|
|
if err == nil {
|
|
break
|
|
}
|
|
if attempt == totalAttempts {
|
|
break
|
|
}
|
|
if shouldRetry != nil && !shouldRetry(err) {
|
|
break
|
|
}
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
|
|
backoff := time.Duration(1<<uint(attempt-1)) * time.Second
|
|
if onRetry != nil {
|
|
onRetry(attempt+1, totalAttempts, backoff, err)
|
|
}
|
|
metadataFlushSleep(ctx, backoff)
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
}
|
|
return err
|
|
}
|