feat(s3/lifecycle): cluster rate-limit allocation (Phase 3)

Admin computes a per-worker share of cluster_deletes_per_second at
ExecuteJob time and ships it to the worker via
ClusterContext.Metadata. The worker reads the share, constructs a
golang.org/x/time/rate.Limiter, and passes it to dailyrun.Run via
cfg.Limiter (Phase 2 already plumbed the field). Phase 5 deletes the
streaming path; until then streaming ignores the cap.

Why allocate at admin: the cluster cap is a single knob operators
care about. Dividing it locally per worker would either need
out-of-band coordination or accept N× the configured budget. Admin
is the only party that knows how many execute-capable workers there
are, so it owns the math.

Admin side (weed/admin/plugin):
- Registry.CountCapableExecutors(jobType) returns the number of
  non-stale workers with CanExecute=true.
- New file cluster_rate_limit.go: decorateClusterContextForJob clones
  the input ClusterContext and injects two metadata keys for
  s3_lifecycle. cloneClusterContext duplicates Metadata so per-job
  decoration doesn't race shared base state.
- executeJobWithExecutor calls the decorator after loading the admin
  config; other job types pass through unchanged.

Worker side (weed/worker/tasks/s3_lifecycle):
- New cluster_rate_limit.go declares the constants both sides agree
  on (admin-config field names, metadata keys). Plain strings on the
  admin side keep weed/admin/plugin free of a dependency on the
  s3_lifecycle worker package; the two sets of constants are pinned
  to identical values and a mismatch would silently disable rate
  limiting.
- handler.go executeDailyReplay reads ClusterContext.Metadata,
  builds a rate.Limiter, and passes it into dailyrun.Config{Limiter}.
  Missing/empty/non-positive values → no limiter (legacy unlimited
  behavior). burst defaults to 2 × rate, clamped to ≥1 to avoid a
  bucket that never refills.
- Admin form gains two fields under "Scope": cluster_deletes_per_second
  (rate, 0 = unlimited) and cluster_deletes_burst (0 = 2 × rate).

Metric:
- New S3LifecycleDispatchLimiterWaitSeconds histogram observes how
  long each Limiter.Wait blocks before a LifecycleDelete RPC.
  Operators tune the cap by reading p95 — near-zero means the cap
  isn't binding, a long tail at 1/rate means it is.

Tests:
- weed/admin/plugin/cluster_rate_limit_test.go: 9 cases covering
  pass-through for non-allocator job types, rps=0 / no-executors
  skip, even sharing, burst sharing, burst=0 omit (worker default
  kicks in), burst floor of 1, no mutation of input metadata, nil
  input.
- weed/worker/tasks/s3_lifecycle/cluster_rate_limit_test.go: 7 cases
  covering nil/empty/missing metadata, non-positive/invalid rate,
  positive rate builds correctly, burst missing defaults to 2× rate,
  tiny rate clamps burst to ≥1.

Build clean. Phase 2 (#9446) and Phase 4 engine (#9447) are the
parents; this branch stacks on Phase 2 since it consumes
dailyrun.Config{Limiter} which lands there.

weed/admin/plugin/cluster_rate_limit.go

@@ -0,0 +1,134 @@
+package plugin
+
+import (
+	"strconv"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+)
+
+// Job types whose ExecuteJobRequest needs per-worker rate-allocation
+// metadata injected. Keyed by the job-type string so plugin.go's
+// generic dispatch path stays job-agnostic.
+//
+// To add a new job type to the share-allocation pipeline: register an
+// entry here that knows how to read its admin-config field(s) and
+// produce the metadata keys/values the worker reads.
+
+// s3LifecycleClusterDeletesPerSecondKey, s3LifecycleClusterDeletesBurstKey,
+// s3LifecycleMetadataDeletesPerSecond, and s3LifecycleMetadataDeletesBurst
+// are the contract between admin and worker. The values must match the
+// constants exported from weed/worker/tasks/s3_lifecycle/cluster_rate_limit.go
+// — duplicated here as plain strings rather than imported so the admin
+// plugin package doesn't pull a dependency on the worker handler
+// package. A mismatch on either side would silently disable rate
+// limiting; tests pin the constants in both packages against the same
+// values.
+const (
+	s3LifecycleJobType                    = "s3_lifecycle"
+	s3LifecycleClusterDeletesPerSecondKey = "cluster_deletes_per_second"
+	s3LifecycleClusterDeletesBurstKey     = "cluster_deletes_burst"
+	s3LifecycleMetadataDeletesPerSecond   = "s3_lifecycle.deletes_per_second"
+	s3LifecycleMetadataDeletesBurst       = "s3_lifecycle.deletes_burst"
+)
+
+// decorateClusterContextForJob returns a new ClusterContext with any
+// per-job-type metadata the admin needs to inject before the
+// ExecuteJobRequest is sent. Returns the input cc unchanged when no
+// decoration applies.
+//
+// Today only s3_lifecycle decorates; the function exists so a future
+// job type's plumbing slots in alongside without touching
+// executeJobWithExecutor.
+func (r *Plugin) decorateClusterContextForJob(cc *plugin_pb.ClusterContext, jobType string, adminConfigValues map[string]*plugin_pb.ConfigValue) *plugin_pb.ClusterContext {
+	if cc == nil {
+		return cc
+	}
+	if jobType != s3LifecycleJobType {
+		return cc
+	}
+	rps := readNonNegativeInt(adminConfigValues, s3LifecycleClusterDeletesPerSecondKey)
+	burst := readNonNegativeInt(adminConfigValues, s3LifecycleClusterDeletesBurstKey)
+	if rps <= 0 {
+		// Operator hasn't configured a cluster cap; nothing to allocate.
+		// The worker treats missing metadata keys as "unlimited," which
+		// is the legacy behavior.
+		return cc
+	}
+	executors := r.registry.CountCapableExecutors(jobType)
+	if executors <= 0 {
+		// No executors means the job won't dispatch at all; metadata
+		// would be discarded. Log so the case is visible in ops.
+		glog.V(2).Infof("decorateClusterContext: %s rps=%d but no execute-capable workers; skipping allocation", jobType, rps)
+		return cc
+	}
+	perWorkerRps := float64(rps) / float64(executors)
+	perWorkerBurst := 0
+	if burst > 0 {
+		perWorkerBurst = burst / executors
+		if perWorkerBurst < 1 {
+			perWorkerBurst = 1
+		}
+	}
+
+	// Clone so we don't mutate the shared base context. The metadata
+	// map is small; a fresh allocation per ExecuteJob is fine.
+	out := cloneClusterContext(cc)
+	if out.Metadata == nil {
+		out.Metadata = map[string]string{}
+	}
+	out.Metadata[s3LifecycleMetadataDeletesPerSecond] = strconv.FormatFloat(perWorkerRps, 'f', -1, 64)
+	if perWorkerBurst > 0 {
+		out.Metadata[s3LifecycleMetadataDeletesBurst] = strconv.Itoa(perWorkerBurst)
+	}
+	glog.V(3).Infof("decorateClusterContext: %s rps=%d burst=%d executors=%d -> per-worker rps=%g burst=%d",
+		jobType, rps, burst, executors, perWorkerRps, perWorkerBurst)
+	return out
+}
+
+// cloneClusterContext returns a shallow-but-safe copy: the top-level
+// fields are reassigned, and the Metadata map is duplicated so the
+// caller can mutate it without racing other consumers of the input.
+// Slices of strings (master/filer/volume/s3 addresses) are copied by
+// reference — those are treated as immutable elsewhere in the codebase.
+func cloneClusterContext(in *plugin_pb.ClusterContext) *plugin_pb.ClusterContext {
+	if in == nil {
+		return nil
+	}
+	out := &plugin_pb.ClusterContext{
+		MasterGrpcAddresses: in.MasterGrpcAddresses,
+		FilerGrpcAddresses:  in.FilerGrpcAddresses,
+		VolumeGrpcAddresses: in.VolumeGrpcAddresses,
+		S3GrpcAddresses:     in.S3GrpcAddresses,
+	}
+	if in.Metadata != nil {
+		out.Metadata = make(map[string]string, len(in.Metadata))
+		for k, v := range in.Metadata {
+			out.Metadata[k] = v
+		}
+	}
+	return out
+}
+
+// readNonNegativeInt reads an int64 admin config value, treating
+// missing fields and non-int kinds as 0. Negative values are clamped
+// to 0 since the AdminConfigForm declares MinValue=0 on both fields.
+func readNonNegativeInt(values map[string]*plugin_pb.ConfigValue, field string) int {
+	v, ok := values[field]
+	if !ok || v == nil {
+		return 0
+	}
+	switch k := v.Kind.(type) {
+	case *plugin_pb.ConfigValue_Int64Value:
+		if k.Int64Value < 0 {
+			return 0
+		}
+		return int(k.Int64Value)
+	case *plugin_pb.ConfigValue_DoubleValue:
+		if k.DoubleValue < 0 {
+			return 0
+		}
+		return int(k.DoubleValue)
+	}
+	return 0
+}

weed/admin/plugin/cluster_rate_limit_test.go

@@ -0,0 +1,148 @@
+package plugin
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// pluginWithExecutors returns a Plugin whose registry contains n
+// non-stale execute-capable workers for jobType. Helper for the
+// allocator tests. Bypasses UpsertFromHello so tests don't have to
+// build a full Hello message.
+func pluginWithExecutors(t *testing.T, jobType string, n int) *Plugin {
+	t.Helper()
+	reg := NewRegistry()
+	now := time.Now()
+	for i := 0; i < n; i++ {
+		id := "worker-" + string(rune('a'+i))
+		reg.sessions[id] = &WorkerSession{
+			WorkerID:    id,
+			LastSeenAt:  now,
+			ConnectedAt: now,
+			Capabilities: map[string]*plugin_pb.JobTypeCapability{
+				jobType: {CanExecute: true},
+			},
+		}
+	}
+	return &Plugin{registry: reg}
+}
+
+// adminConfig builds an int64 admin config map for the given fields.
+func adminConfig(pairs ...interface{}) map[string]*plugin_pb.ConfigValue {
+	if len(pairs)%2 != 0 {
+		panic("adminConfig expects key/value pairs")
+	}
+	out := map[string]*plugin_pb.ConfigValue{}
+	for i := 0; i < len(pairs); i += 2 {
+		key := pairs[i].(string)
+		switch v := pairs[i+1].(type) {
+		case int:
+			out[key] = &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(v)}}
+		case int64:
+			out[key] = &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: v}}
+		case float64:
+			out[key] = &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: v}}
+		default:
+			panic("adminConfig: unsupported value type")
+		}
+	}
+	return out
+}
+
+func TestDecorateClusterContext_NonS3LifecycleIsPassThrough(t *testing.T) {
+	// Any job type other than s3_lifecycle gets the input cc back
+	// unchanged. Future allocators add their own branch; the default
+	// is pass-through.
+	r := pluginWithExecutors(t, s3LifecycleJobType, 4)
+	in := &plugin_pb.ClusterContext{Metadata: map[string]string{"unrelated": "v"}}
+	out := r.decorateClusterContextForJob(in, "some_other_job", adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100))
+	assert.Same(t, in, out, "non-allocator job type must return the same pointer")
+}
+
+func TestDecorateClusterContext_RpsZeroSkipsAllocation(t *testing.T) {
+	// rps=0 means "operator hasn't configured a cap"; the worker
+	// treats missing keys as unlimited. We must NOT inject any
+	// metadata (in particular, not "0") because that would force the
+	// worker into a no-throughput state on a misconfigured cluster.
+	r := pluginWithExecutors(t, s3LifecycleJobType, 4)
+	in := &plugin_pb.ClusterContext{}
+	out := r.decorateClusterContextForJob(in, s3LifecycleJobType, adminConfig(s3LifecycleClusterDeletesPerSecondKey, 0))
+	if out.Metadata != nil {
+		_, hasRps := out.Metadata[s3LifecycleMetadataDeletesPerSecond]
+		assert.False(t, hasRps, "rps=0 must not write a deletes_per_second key")
+	}
+}
+
+func TestDecorateClusterContext_NoExecutorsSkipsAllocation(t *testing.T) {
+	r := pluginWithExecutors(t, s3LifecycleJobType, 0)
+	in := &plugin_pb.ClusterContext{}
+	out := r.decorateClusterContextForJob(in, s3LifecycleJobType, adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100))
+	if out.Metadata != nil {
+		_, hasRps := out.Metadata[s3LifecycleMetadataDeletesPerSecond]
+		assert.False(t, hasRps, "0 executors must not write share metadata (would divide by zero)")
+	}
+}
+
+func TestDecorateClusterContext_SharedEvenly(t *testing.T) {
+	r := pluginWithExecutors(t, s3LifecycleJobType, 4)
+	in := &plugin_pb.ClusterContext{}
+	out := r.decorateClusterContextForJob(in, s3LifecycleJobType, adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100))
+	require.NotNil(t, out.Metadata)
+	assert.Equal(t, "25", out.Metadata[s3LifecycleMetadataDeletesPerSecond], "100 / 4 executors = 25/worker")
+}
+
+func TestDecorateClusterContext_BurstShared(t *testing.T) {
+	r := pluginWithExecutors(t, s3LifecycleJobType, 2)
+	in := &plugin_pb.ClusterContext{}
+	out := r.decorateClusterContextForJob(in, s3LifecycleJobType,
+		adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100, s3LifecycleClusterDeletesBurstKey, 20))
+	require.NotNil(t, out.Metadata)
+	assert.Equal(t, "50", out.Metadata[s3LifecycleMetadataDeletesPerSecond])
+	assert.Equal(t, "10", out.Metadata[s3LifecycleMetadataDeletesBurst])
+}
+
+func TestDecorateClusterContext_BurstZeroOmitsKey(t *testing.T) {
+	// burst=0 means "let the worker default it." Don't write the key —
+	// the worker's parsePositiveInt would then take the unset path
+	// and compute 2 × rps automatically.
+	r := pluginWithExecutors(t, s3LifecycleJobType, 4)
+	in := &plugin_pb.ClusterContext{}
+	out := r.decorateClusterContextForJob(in, s3LifecycleJobType,
+		adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100, s3LifecycleClusterDeletesBurstKey, 0))
+	_, hasBurst := out.Metadata[s3LifecycleMetadataDeletesBurst]
+	assert.False(t, hasBurst, "burst=0 must NOT write the burst key (worker default kicks in)")
+}
+
+func TestDecorateClusterContext_BurstFloorIsOneWhenDividesBelowOne(t *testing.T) {
+	// burst=1 across 4 workers would round to 0; clamp to 1 so the
+	// limiter doesn't become "single-token bucket that never refills."
+	r := pluginWithExecutors(t, s3LifecycleJobType, 4)
+	in := &plugin_pb.ClusterContext{}
+	out := r.decorateClusterContextForJob(in, s3LifecycleJobType,
+		adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100, s3LifecycleClusterDeletesBurstKey, 1))
+	assert.Equal(t, "1", out.Metadata[s3LifecycleMetadataDeletesBurst])
+}
+
+func TestDecorateClusterContext_DoesNotMutateInput(t *testing.T) {
+	// The same base ClusterContext is shared across many parallel
+	// ExecuteJob calls. The decorator must produce a fresh map so it
+	// can't race / leak per-job metadata into the base.
+	r := pluginWithExecutors(t, s3LifecycleJobType, 4)
+	baseMeta := map[string]string{"existing": "value"}
+	in := &plugin_pb.ClusterContext{Metadata: baseMeta}
+	_ = r.decorateClusterContextForJob(in, s3LifecycleJobType,
+		adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100))
+	_, leaked := baseMeta[s3LifecycleMetadataDeletesPerSecond]
+	assert.False(t, leaked, "decorator must not mutate the input metadata map")
+	assert.Equal(t, "value", baseMeta["existing"])
+}
+
+func TestDecorateClusterContext_NilInputPassesThrough(t *testing.T) {
+	r := pluginWithExecutors(t, s3LifecycleJobType, 4)
+	out := r.decorateClusterContextForJob(nil, s3LifecycleJobType, adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100))
+	assert.Nil(t, out)
+}

weed/admin/plugin/plugin.go

@@ -649,6 +649,12 @@ func (r *Plugin) executeJobWithExecutor(
 		return nil, err
 	}
 
+	// Apply per-job-type cluster-allocation decoration (e.g. s3_lifecycle
+	// divides cluster_deletes_per_second by the worker count and ships
+	// the share via ClusterContext.Metadata). No-op for job types
+	// without an allocator registered.
+	clusterContext = r.decorateClusterContextForJob(clusterContext, job.JobType, adminConfigValues)
+
 	completedCh := make(chan *plugin_pb.JobCompleted, 1)
 	r.pendingExecutionMu.Lock()
 	r.pendingExecution[requestID] = completedCh

weed/admin/plugin/registry.go

@@ -142,6 +142,31 @@ func (r *Registry) HasCapableWorker(jobType string) bool {
 	return false
 }
 
+// CountCapableExecutors returns the number of non-stale workers that
+// can EXECUTE the given job type. Used by per-job-type cluster
+// allocators (e.g. the s3_lifecycle delete-rate divider) to compute a
+// per-worker share at dispatch time. Returns 0 when no executor is
+// available — callers should treat that as "skip allocation" rather
+// than dividing by zero.
+func (r *Registry) CountCapableExecutors(jobType string) int {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	now := time.Now()
+	n := 0
+	for _, session := range r.sessions {
+		if r.isSessionStaleLocked(session, now) {
+			continue
+		}
+		capability := session.Capabilities[jobType]
+		if capability == nil || !capability.CanExecute {
+			continue
+		}
+		n++
+	}
+	return n
+}
+
 // DetectableJobTypes returns sorted job types that currently have at least one detect-capable worker.
 func (r *Registry) DetectableJobTypes() []string {
 	r.mu.RLock()