mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-17 20:10:51 +02:00
Chris Lu
4bf27278fa
* operation: bound upload retries and honor context cancellation retriedUploadData hardcoded 3 attempts and an uninterruptible backoff sleep. A synchronous replica write to a dead host therefore paid the full dial timeout three times over before failing. Add UploadOption.MaxAttempts (<=0 keeps the default of 3) so callers can cap attempts, and make the loop return as soon as the context is cancelled so an abandoned upload unwinds instead of retrying. * topology: fail replica writes fast when a replica is unreachable DistributedOperation already returns on the first error, but a single dead replica is itself the slow result: its goroutine retries the upload three times through the dial timeout (~30s) before any error surfaces, stalling the originating client write the whole time. Make the replica write a single attempt (MaxAttempts=1) so a dead replica fails after one dial timeout instead of three, and thread a context into DistributedOperation that is cancelled once the outcome is decided, so a healthy replica is no longer held hostage by one stalled in a dial. The originating client write is what retries. * topology: keep replica deletes off the client request context ReplicatedDelete runs after the local needle is already deleted. Driving the replica deletes off r.Context() means a client disconnect cancels them and orphans needles on the replicas, so use a background context. * operation, topology: trim comments on the replica fail-fast path
56 lines
1.5 KiB
Go
56 lines
1.5 KiB
Go
package topology
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/operation"
|
|
)
|
|
|
|
// TestDistributedOperationCancelsSiblingsOnFirstError verifies that once one
|
|
// replica fails, an outstanding replica still stalled in a dial timeout is
|
|
// cancelled rather than gating the caller until it times out.
|
|
func TestDistributedOperationCancelsSiblingsOnFirstError(t *testing.T) {
|
|
locations := []operation.Location{{Url: "fast"}, {Url: "slow"}}
|
|
cancelled := make(chan struct{}, 1)
|
|
|
|
start := time.Now()
|
|
err := DistributedOperation(context.Background(), locations, func(ctx context.Context, location operation.Location) error {
|
|
if location.Url == "fast" {
|
|
return errors.New("connection refused")
|
|
}
|
|
// slow: a replica stalled in a dial timeout
|
|
select {
|
|
case <-ctx.Done():
|
|
cancelled <- struct{}{}
|
|
return ctx.Err()
|
|
case <-time.After(10 * time.Second):
|
|
return nil
|
|
}
|
|
})
|
|
|
|
if err == nil {
|
|
t.Fatal("expected an error from the fast-failing replica")
|
|
}
|
|
if elapsed := time.Since(start); elapsed > 2*time.Second {
|
|
t.Fatalf("did not fail fast: took %v", elapsed)
|
|
}
|
|
select {
|
|
case <-cancelled:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("slow replica was not cancelled after the first error")
|
|
}
|
|
}
|
|
|
|
func TestDistributedOperationEmpty(t *testing.T) {
|
|
err := DistributedOperation(context.Background(), nil, func(ctx context.Context, location operation.Location) error {
|
|
t.Fatal("op should not be called when there are no locations")
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected nil for no locations, got %v", err)
|
|
}
|
|
}
|