Files
seaweedfs/weed/s3api/s3lifecycle/engine/engine.go
T
74ed12d6b3 s3 lifecycle: replay only the rules that fit the retention window (#11789)
* s3 lifecycle: replay only the rules that fit the retention window

runShard handed drainShardEvents the full snapshot, so rules the retention
window assigns to the walk were also matched during meta-log replay. Their
matches are not due yet, and drainShardEvents freezes the cursor at the first
event with a not-yet-due match. With a 3-day and a 30-day expiration rule on one
bucket and RetentionWindow of 8 days, every shard's cursor stopped at the first
30-day object's PUT and stayed there until that object came due. Each pass
then re-read the log from that point and re-dispatched every 3-day expiration
already handled, which surfaces as NOOP_RESOLVED dispatches, a growing
cursor_lag_max, and passes that still report status=ok.

Snapshot.ReplayView drops the replay-kind actions promoted to the walk and
keeps every other action as compiled, and the drain uses it. Promoted rules
are served by the walk as before. RulesForShard, PromotedHash and ReplayView
share one isPromoted predicate. With RetentionWindow unset no active rule is
promoted, so behaviour without a configured window is unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3 lifecycle: scale the replay-view test with DaysToDuration

The test took its retention window from DaysToDuration but dated its PUTs
in hours, so under the s3tests build tag (one lifecycle day = 10s) every
event fell before the seeded cursor and replay never ran. Express the
event ages in DaysToDuration too, so the test exercises replay at both
scales.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3 lifecycle: rehabilitate scan_only mode in the replay drain view

ReplayView shared the base CompiledAction pointers, so a replay-kind
rule locked into ModeScanOnly by a prior compile stayed invisible to
router.Route's event_driven gate even when the runtime retention window
kept it in the replay partition - and it is absent from the walk view
too, so the rule would be served by neither.

Clone non-promoted replay-kind actions the way RulesForShard does
(Mode forced to event_driven), reactivating the scan_only case whose
bootstrap rendezvous already completed, while preserving the base
active bit for genuinely pre-bootstrap actions. Disabled actions are
excluded for parity with the other views, and DESIGN.md now documents
the view.

* s3 lifecycle: keep drain activation observable, require walker with retention

ReplayView now shares pointers for event-driven actions so a mid-pass
MarkActive flip remains visible to the drain; only a scan_only-locked
action is cloned to event_driven, and reactivated only when its
bootstrap rendezvous completed (tracked via a new bootstrapComplete
field on CompiledAction).

validate() rejects RetentionWindow without Walker: promoted rules
would lose their only dispatch path while the drain cursor still
advanced past their events.

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Chris Lu <chris.lu@gmail.com>
2026-10-17 15:24:35 -07:00

158 lines
4.3 KiB
Go

// Package engine compiles per-bucket lifecycle rules into a CompiledAction
// snapshot. One XML <Rule> expands into N compiled actions (one per populated
// action sub-element); each is keyed by ActionKey{bucket, rule_hash,
// action_kind} so sibling actions schedule and degrade independently.
package engine
import (
"sync/atomic"
"time"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
)
type Engine struct {
current atomic.Pointer[Snapshot]
}
func New() *Engine {
e := &Engine{}
e.current.Store(&Snapshot{actions: map[s3lifecycle.ActionKey]*CompiledAction{}})
return e
}
func (e *Engine) Snapshot() *Snapshot { return e.current.Load() }
// Snapshot fields are append-only after Compile except CompiledAction.engineState,
// which transitions inactive -> active atomically via markActive.
type Snapshot struct {
id uint64
buckets map[string]*BucketIndex
actions map[s3lifecycle.ActionKey]*CompiledAction
allActionsSorted []*CompiledAction
// Routing indexes hold every ActionKey by mode regardless of activation;
// dispatch filters on IsActive().
originalDelayGroups map[time.Duration][]s3lifecycle.ActionKey
predicateActions []s3lifecycle.ActionKey
dateActions map[s3lifecycle.ActionKey]time.Time
}
func (s *Snapshot) SnapshotID() uint64 { return s.id }
type BucketIndex struct {
bucket string
versioned bool
actionKeys []s3lifecycle.ActionKey
}
type CompiledAction struct {
Rule *s3lifecycle.Rule
Bucket string
Key s3lifecycle.ActionKey
Delay time.Duration
PredicateSensitive bool
Mode RuleMode
bootstrapComplete bool
engineState atomic.Uint32
}
const (
engineStateInactive uint32 = 0
engineStateActive uint32 = 1
)
func (a *CompiledAction) IsActive() bool {
return a.engineState.Load() == engineStateActive
}
func (a *CompiledAction) markActive() {
a.engineState.Store(engineStateActive)
}
// MarkActive mirrors the in-memory hint after a durable bootstrap_complete
// write. No-op if the key isn't in the snapshot.
func (s *Snapshot) MarkActive(key s3lifecycle.ActionKey) {
if a, ok := s.actions[key]; ok {
a.markActive()
}
}
func (s *Snapshot) Action(key s3lifecycle.ActionKey) *CompiledAction {
return s.actions[key]
}
// AllActions: caller must not mutate.
func (s *Snapshot) AllActions() []*CompiledAction {
return s.allActionsSorted
}
// OriginalDelayGroups / PredicateActions / DateActions return defensive copies.
func (s *Snapshot) OriginalDelayGroups() map[time.Duration][]s3lifecycle.ActionKey {
out := make(map[time.Duration][]s3lifecycle.ActionKey, len(s.originalDelayGroups))
for d, keys := range s.originalDelayGroups {
copied := make([]s3lifecycle.ActionKey, len(keys))
copy(copied, keys)
out[d] = copied
}
return out
}
func (s *Snapshot) PredicateActions() []s3lifecycle.ActionKey {
out := make([]s3lifecycle.ActionKey, len(s.predicateActions))
copy(out, s.predicateActions)
return out
}
func (s *Snapshot) DateActions() map[s3lifecycle.ActionKey]time.Time {
out := make(map[s3lifecycle.ActionKey]time.Time, len(s.dateActions))
for k, v := range s.dateActions {
out[k] = v
}
return out
}
func (s *Snapshot) BucketVersioned(bucket string) bool {
if bi, ok := s.buckets[bucket]; ok {
return bi.versioned
}
return false
}
// BucketPrefixes returns the distinct rule prefixes of the bucket's
// active actions in this view. An empty result means no active action
// can match any entry in the bucket; a "" element means some rule
// matches every key.
func (s *Snapshot) BucketPrefixes(bucket string) []string {
bi := s.buckets[bucket]
if bi == nil {
return nil
}
seen := map[string]bool{}
var out []string
for _, k := range bi.actionKeys {
a := s.actions[k]
if a == nil || !a.IsActive() || a.Mode == ModeDisabled || a.Rule == nil {
continue
}
if !seen[a.Rule.Prefix] {
seen[a.Rule.Prefix] = true
out = append(out, a.Rule.Prefix)
}
}
return out
}
// BucketActionKeys returns a defensive copy of the action keys for bucket,
// or nil if the bucket has no compiled actions in this snapshot.
func (s *Snapshot) BucketActionKeys(bucket string) []s3lifecycle.ActionKey {
bi, ok := s.buckets[bucket]
if !ok {
return nil
}
out := make([]s3lifecycle.ActionKey, len(bi.actionKeys))
copy(out, bi.actionKeys)
return out
}