wdclient, dailyrun: add equal jitter to retry backoff (#9737)

* wdclient, dailyrun: add equal jitter to retry backoff

Prevents thundering-herd retries when many clients recover from a
transient failure at the same instant (e.g., filer restart, network
partition healing).

Uses equal jitter: wait in [d/2, d) instead of deterministic d.
This bounds the maximum wait while still desynchronizing clients.

Files:
- weed/wdclient/filer_client.go   (LookupVolumeIds retry loop)
- weed/s3api/s3lifecycle/dailyrun/dispatch.go (dispatchWithRetry)

Tests added for bounds, zero/negative inputs, and distribution sanity.

Closes #9735

* wdclient: honor ctx cancellation during LookupVolumeIds backoff

---------

Co-authored-by: Mohamed Chorfa <mohamed.chorfa@thalesgroup.com>
Co-authored-by: Chris Lu <chris.lu@gmail.com>

weed/s3api/s3lifecycle/dailyrun/dispatch.go

@@ -3,6 +3,7 @@ package dailyrun
 import (
 	"context"
 	"errors"
+	"math/rand"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
@@ -52,7 +53,7 @@ func dispatchWithRetry(ctx context.Context, client LifecycleClient, m router.Mat
 		select {
 		case <-ctx.Done():
 			return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, ctx.Err()
-		case <-time.After(backoff):
+		case <-time.After(jitter(backoff)):
 		}
 		backoff *= 2
 		if backoff > transportRetryMax {
@@ -62,6 +63,19 @@ func dispatchWithRetry(ctx context.Context, client LifecycleClient, m router.Mat
 	return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, lastErr
 }
 
+// jitter returns a duration in the range [d/2, d) using equal jitter.
+// Prevents thundering herds when many daily-run workers retry simultaneously.
+func jitter(d time.Duration) time.Duration {
+	if d <= 0 {
+		return 0
+	}
+	half := d / 2
+	if half <= 0 {
+		return d
+	}
+	return half + time.Duration(rand.Int63n(int64(half)))
+}
+
 // buildDeleteRequest constructs the LifecycleDelete RPC payload for a
 // router Match. Mirrors dispatcher.dispatchOne's request shape — both
 // targets the same server-side handler and the proto encoding must

weed/s3api/s3lifecycle/dailyrun/jitter_test.go

@@ -0,0 +1,44 @@
+package dailyrun
+
+import (
+	"testing"
+	"time"
+)
+
+func TestJitterBounds(t *testing.T) {
+	cases := []time.Duration{
+		200 * time.Millisecond,
+		1 * time.Second,
+		5 * time.Second,
+	}
+
+	for _, d := range cases {
+		for i := 0; i < 100; i++ {
+			j := jitter(d)
+			if j < d/2 {
+				t.Errorf("jitter(%v) = %v, below lower bound %v", d, j, d/2)
+			}
+			if j >= d {
+				t.Errorf("jitter(%v) = %v, at or above upper bound %v", d, j, d)
+			}
+		}
+	}
+}
+
+func TestJitterZeroAndNegative(t *testing.T) {
+	if j := jitter(0); j != 0 {
+		t.Errorf("jitter(0) = %v, want 0", j)
+	}
+	if j := jitter(-1 * time.Second); j != 0 {
+		t.Errorf("jitter(-1s) = %v, want 0", j)
+	}
+}
+
+func TestJitterTinyDuration(t *testing.T) {
+	// When d < 2, half == 0 and rand.Int63n(0) panics.
+	// We should return d unmodified in that case.
+	j := jitter(1 * time.Nanosecond)
+	if j != 1*time.Nanosecond {
+		t.Errorf("jitter(1ns) = %v, want 1ns", j)
+	}
+}