seaweedfs/weed/s3api/s3lifecycle/dailyrun/process_matches_test.go

package dailyrun

import (
	"context"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	dto "github.com/prometheus/client_model/go"
	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
	"github.com/seaweedfs/seaweedfs/weed/stats"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// recordingClient captures every LifecycleDelete request the
// daily-run path emits. Default outcome is DONE; tests that need
// other outcomes set responses by index, or outcomeByObject when
// shards dispatch concurrently and call order isn't fixed.
type recordingClient struct {
	mu              sync.Mutex
	requests        []*s3_lifecycle_pb.LifecycleDeleteRequest
	responses       []s3_lifecycle_pb.LifecycleDeleteOutcome
	outcomeByObject map[string]s3_lifecycle_pb.LifecycleDeleteOutcome
	calls           atomic.Int32
}

func (c *recordingClient) LifecycleDelete(_ context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
	c.mu.Lock()
	c.requests = append(c.requests, req)
	idx := int(c.calls.Add(1)) - 1
	byObject, pinned := c.outcomeByObject[req.ObjectPath]
	c.mu.Unlock()
	if pinned {
		return &s3_lifecycle_pb.LifecycleDeleteResponse{Outcome: byObject}, nil
	}
	out := s3_lifecycle_pb.LifecycleDeleteOutcome_DONE
	if idx < len(c.responses) {
		out = c.responses[idx]
	}
	return &s3_lifecycle_pb.LifecycleDeleteResponse{Outcome: out}, nil
}

func (c *recordingClient) seenObjects() []string {
	c.mu.Lock()
	defer c.mu.Unlock()
	out := make([]string, 0, len(c.requests))
	for _, r := range c.requests {
		out = append(out, r.ObjectPath)
	}
	return out
}

// Phase 2's processMatches must NOT use a per-rule done flag.
// routePointerTransitionExpand can return multiple matches for the
// same ActionKey on different objects, each with its own
// SuccessorModTime from a distinct demoting event. A not-yet-due
// sibling must never gate a sibling that's already past its DueTime.
// Pin that here.
func TestProcessMatches_NotYetDueDoesNotSuppressDueOnSameRule(t *testing.T) {
	runNow := time.Now()
	rh := [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
	rule := s3lifecycle.ActionKey{
		Bucket:     "b",
		RuleHash:   rh,
		ActionKind: s3lifecycle.ActionKindNoncurrentDays,
	}
	// Three matches for the same ActionKey, different objects, mixed
	// due-times. Order intentionally puts the not-yet-due match first
	// so a buggy per-rule done flag would suppress the others.
	matches := []router.Match{
		{Key: rule, Bucket: "b", ObjectKey: "future-sibling", DueTime: runNow.Add(48 * time.Hour)},
		{Key: rule, Bucket: "b", ObjectKey: "due-sibling-a", DueTime: runNow.Add(-48 * time.Hour)},
		{Key: rule, Bucket: "b", ObjectKey: "due-sibling-b", DueTime: runNow.Add(-24 * time.Hour)},
	}
	client := &recordingClient{}
	cfg := Config{Client: client}
	skipped, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{TsNs: runNow.UnixNano()}, matches)
	require.NoError(t, err)
	require.False(t, halted)
	require.True(t, skipped, "at least one future-DueTime match must flag skippedAny")

	// Both due siblings must have been dispatched.
	seen := client.seenObjects()
	assert.ElementsMatch(t, []string{"due-sibling-a", "due-sibling-b"}, seen,
		"due matches must dispatch regardless of where the future-sibling sits in the slice")
}

func TestProcessMatches_OrderingDoesNotMatter(t *testing.T) {
	runNow := time.Now()
	rh := [8]byte{0xaa}
	rule := s3lifecycle.ActionKey{Bucket: "b", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}
	matches := []router.Match{
		{Key: rule, Bucket: "b", ObjectKey: "a", DueTime: runNow.Add(-2 * time.Hour)},
		{Key: rule, Bucket: "b", ObjectKey: "b", DueTime: runNow.Add(-1 * time.Hour)},
		{Key: rule, Bucket: "b", ObjectKey: "future", DueTime: runNow.Add(time.Hour)},
	}
	client := &recordingClient{}
	cfg := Config{Client: client}
	skipped, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{}, matches)
	require.NoError(t, err)
	require.False(t, halted)
	require.True(t, skipped)
	assert.ElementsMatch(t, []string{"a", "b"}, client.seenObjects())
}

func TestProcessMatches_HaltOnServerOutcomeStopsRemaining(t *testing.T) {
	runNow := time.Now()
	rh := [8]byte{0xbb}
	rule := s3lifecycle.ActionKey{Bucket: "b", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}
	matches := []router.Match{
		{Key: rule, Bucket: "b", ObjectKey: "ok", DueTime: runNow.Add(-time.Hour)},
		{Key: rule, Bucket: "b", ObjectKey: "blocked", DueTime: runNow.Add(-time.Hour)},
		{Key: rule, Bucket: "b", ObjectKey: "would-be-next", DueTime: runNow.Add(-time.Hour)},
	}
	client := &recordingClient{responses: []s3_lifecycle_pb.LifecycleDeleteOutcome{
		s3_lifecycle_pb.LifecycleDeleteOutcome_DONE,
		s3_lifecycle_pb.LifecycleDeleteOutcome_BLOCKED,
		s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, // would be a bug if reached
	}}
	cfg := Config{Client: client}
	_, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{}, matches)
	require.NoError(t, err)
	require.True(t, halted, "BLOCKED outcome must halt the event loop")
	assert.Equal(t, []string{"ok", "blocked"}, client.seenObjects(),
		"would-be-next must NOT be dispatched after the halt")
}

func TestProcessMatches_EmptyMatchesIsNoop(t *testing.T) {
	runNow := time.Now()
	client := &recordingClient{}
	cfg := Config{Client: client}
	skipped, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{}, nil)
	require.NoError(t, err)
	require.False(t, halted)
	require.False(t, skipped)
	assert.Empty(t, client.seenObjects())
}

func TestProcessMatches_AllDueNoSkippedFlag(t *testing.T) {
	// All matches past their DueTime — skippedAny must be false so the
	// caller can advance the cursor past this event.
	runNow := time.Now()
	rh := [8]byte{0xcc}
	rule := s3lifecycle.ActionKey{Bucket: "b", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}
	matches := []router.Match{
		{Key: rule, Bucket: "b", ObjectKey: "a", DueTime: runNow.Add(-time.Hour)},
		{Key: rule, Bucket: "b", ObjectKey: "b", DueTime: runNow.Add(-30 * time.Minute)},
	}
	client := &recordingClient{}
	cfg := Config{Client: client}
	skipped, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{}, matches)
	require.NoError(t, err)
	require.False(t, halted)
	require.False(t, skipped, "every match was past DueTime; skippedAny must be false")
}

func TestProcessMatches_DispatchCounterIncrements(t *testing.T) {
	// Pin that dispatched matches increment S3LifecycleDispatchCounter
	// with the outcome label so a refactor doesn't silently drop the
	// observability hook. Use a bucket/kind no other test touches to
	// keep the read-after-write stable.
	runNow := time.Now()
	rh := [8]byte{0xfe}
	rule := s3lifecycle.ActionKey{Bucket: "metrics-pin-bkt", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}
	matches := []router.Match{
		{Key: rule, Bucket: "metrics-pin-bkt", ObjectKey: "obj", DueTime: runNow.Add(-time.Hour)},
	}
	client := &recordingClient{} // default DONE
	before := dispatchCounterValue("metrics-pin-bkt", "expiration_days", "DONE")
	// Delete the label row on exit so this test doesn't leak into the
	// in-process Prometheus registry that other tests share.
	defer stats.S3LifecycleDispatchCounter.DeleteLabelValues("metrics-pin-bkt", "expiration_days", "DONE")
	cfg := Config{Client: client}
	_, _, err := processMatches(context.Background(), cfg, runNow, &reader.Event{}, matches)
	require.NoError(t, err)
	after := dispatchCounterValue("metrics-pin-bkt", "expiration_days", "DONE")
	assert.Equal(t, before+1, after, "DONE outcome must increment the dispatch counter")
}

// dispatchCounterValue reads the current value of the shared
// S3LifecycleDispatchCounter for the given (bucket, kind, outcome).
func dispatchCounterValue(bucket, kind, outcome string) float64 {
	m := stats.S3LifecycleDispatchCounter.WithLabelValues(bucket, kind, outcome)
	var pm dto.Metric
	if err := m.Write(&pm); err != nil {
		return 0
	}
	if pm.Counter == nil {
		return 0
	}
	return pm.Counter.GetValue()
}