seaweedfs/weed/s3api/s3lifecycle/dailyrun/walker_dispatcher.go

package dailyrun

import (
	"context"
	"fmt"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
	"github.com/seaweedfs/seaweedfs/weed/stats"
	"golang.org/x/time/rate"
)

// WalkerDispatcher adapts LifecycleClient to bootstrap.Dispatcher so
// the Phase 4b walker can drive the same LifecycleDelete RPC the
// meta-log replay path uses, with the listed entry's identity as the
// CAS witness: an object rewritten between listing and dispatch
// resolves as STALE_IDENTITY instead of being deleted.
type WalkerDispatcher struct {
	Client LifecycleClient
	// Limiter throttles LifecycleDelete dispatch under the cluster's
	// shared deletes/sec budget. Should be the same *rate.Limiter the
	// daily-run's processMatches uses so the walker and replay paths
	// can't combine to burst past the cap. nil disables throttling.
	Limiter *rate.Limiter
}

// Compile-time check.
var _ bootstrap.Dispatcher = (*WalkerDispatcher)(nil)

// Delete classifies non-DONE server outcomes as errors so the walker
// halts and the caller persists progress under the bootstrap
// checkpoint rather than silently skipping objects.
func (d *WalkerDispatcher) Delete(ctx context.Context, action *engine.CompiledAction, entry *bootstrap.Entry) error {
	if d == nil || d.Client == nil {
		return fmt.Errorf("walker dispatch: nil client")
	}
	if action == nil || entry == nil {
		return fmt.Errorf("walker dispatch: nil action or entry")
	}
	objectPath := entry.Path
	if entry.IsMPUInit {
		// Rule-prefix matching uses DestKey (the user's intended
		// object key); dispatch uses entry.Path (.uploads/<id>),
		// which is what the server's ABORT_MPU handler expects in
		// req.ObjectPath — it strips the .uploads/ prefix to get
		// the upload id and reads the init record from that
		// directory. DestKey is the dispatch ANTI-pattern here: it
		// looks like a regular object path, the server's check for
		// the .uploads/ prefix fails, and the dispatch comes back
		// as BLOCKED FATAL_EVENT_ERROR.
		if entry.DestKey == "" {
			return fmt.Errorf("walker dispatch: MPU init entry with empty DestKey: %s", entry.Path)
		}
	}
	rh := action.Key.RuleHash
	req := &s3_lifecycle_pb.LifecycleDeleteRequest{
		Bucket:           action.Bucket,
		ObjectPath:       objectPath,
		VersionId:        entry.VersionID,
		RuleHash:         rh[:],
		ActionKind:       toProtoActionKind(action.Key.ActionKind),
		ExpectedIdentity: toProtoIdentity(entry.Identity),
	}
	kindLabel := action.Key.ActionKind.String()
	if d.Limiter != nil {
		waitStart := time.Now()
		if waitErr := d.Limiter.Wait(ctx); waitErr != nil {
			return fmt.Errorf("walker dispatch %s/%s %s: limiter: %w", action.Bucket, objectPath, action.Key.ActionKind, waitErr)
		}
		stats.S3LifecycleDispatchLimiterWaitSeconds.Observe(time.Since(waitStart).Seconds())
	}
	resp, err := d.Client.LifecycleDelete(ctx, req)
	if err != nil {
		// "RPC_ERROR" matches the streaming dispatcher and processMatches
		// labels so transport-class failures aggregate under one key.
		stats.S3LifecycleDispatchCounter.WithLabelValues(action.Bucket, kindLabel, "RPC_ERROR").Inc()
		return fmt.Errorf("walker dispatch %s/%s %s: %w", action.Bucket, objectPath, action.Key.ActionKind, err)
	}
	if resp == nil {
		// A misbehaving server stub returning (nil, nil) would panic on
		// the switch below. Bucket under RPC_ERROR rather than a new
		// label — operationally it's the same class of failure.
		stats.S3LifecycleDispatchCounter.WithLabelValues(action.Bucket, kindLabel, "RPC_ERROR").Inc()
		return fmt.Errorf("walker dispatch %s/%s %s: nil response", action.Bucket, objectPath, action.Key.ActionKind)
	}
	stats.S3LifecycleDispatchCounter.WithLabelValues(action.Bucket, kindLabel, resp.Outcome.String()).Inc()
	switch resp.Outcome {
	case s3_lifecycle_pb.LifecycleDeleteOutcome_DONE,
		s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED,
		s3_lifecycle_pb.LifecycleDeleteOutcome_SKIPPED_OBJECT_LOCK:
		return nil
	default:
		// RETRY_LATER / BLOCKED / UNSPECIFIED: surface as error so the
		// walk halts at this entry and resumes from
		// Checkpoint.LastScannedPath on the next run.
		return fmt.Errorf("walker dispatch %s/%s %s: outcome=%s reason=%s",
			action.Bucket, objectPath, action.Key.ActionKind, resp.Outcome, resp.Reason)
	}
}