mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-21 13:34:26 +02:00
* fix(s3/lifecycle): reader handles bare /buckets parent and pre-normalizes prefix extractBucketKey accepted /buckets/ but rejected /buckets (no trailing slash); some delete events emit the bare form, so bucket-root events were silently dropped. Pre-normalize BucketsPath once on Run instead of recomputing per event. * perf(s3/lifecycle): pool sha256 hashers in ShardID ShardID runs on every meta-log event before the shard filter; a fresh sha256.New per call produces measurable allocator pressure under load. sync.Pool reuses hashers across calls. * fix(s3/lifecycle): router skips hard deletes and missing-attribute events A hard delete carries no schedule-relevant state — Expiration would hit NOOP_RESOLVED at dispatch and ExpiredObjectDeleteMarker fires from a Create on the latest version. Skip rather than burn a schedule slot. Missing Attributes leaves ModTime at year 0001, which makes ExpirationDays fire immediately at dispatch. Skip the event instead. Drop the unused 'versioned' parameter from buildObjectInfo; the dispatcher's identity-CAS handles version drift in Phase 5. * fix(s3/lifecycle): EntryIdentity.MtimeNs holds true nanoseconds Both computeEntryIdentity (server) and buildIdentity (router) wrote entry.Attributes.Mtime (seconds) into a field named MtimeNs. The CAS worked because both sides agreed, but the encoding contradicted the field name and would break if either side later started using true nanoseconds. Combine Mtime*1e9 + the FuseAttributes.MtimeNs nanosecond component on both sides; the test was updated to match. * fix(s3/lifecycle): dispatcher distinguishes ctx cancel from transport errors A canceled or deadline-exceeded RPC is shutdown, not a transport failure: re-queue the Match at its original DueTime with no retry-budget burn so a quick restart can't escalate it to BLOCKED. * fix(s3/lifecycle): reader fallback prefix normalization mirrors Run The fallback path that builds prefix from r.BucketsPath when bucketsPathSlash is empty (test-only entry into extractBucketKey) was appending an unconditional '/', producing '//' if BucketsPath already ended with one. Use the same normalization Run does. * fix(s3/lifecycle): ObjectInfo.ModTime carries the nanosecond component ModTime dropped FuseAttributes.MtimeNs, leaving ExpirationDays one nanosecond off relative to EntryIdentity.MtimeNs. Pass both to time.Unix so the precision matches the CAS witness.
198 lines
6.4 KiB
Go
198 lines
6.4 KiB
Go
package router
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader"
|
|
)
|
|
|
|
func compileWith(rule *s3lifecycle.Rule, prior map[s3lifecycle.ActionKey]engine.PriorState) *engine.Snapshot {
|
|
e := engine.New()
|
|
return e.Compile([]engine.CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
|
engine.CompileOptions{PriorStates: prior})
|
|
}
|
|
|
|
func activatedPrior(rule *s3lifecycle.Rule) map[s3lifecycle.ActionKey]engine.PriorState {
|
|
prior := map[s3lifecycle.ActionKey]engine.PriorState{}
|
|
hash := s3lifecycle.RuleHash(rule)
|
|
for _, k := range s3lifecycle.RuleActionKinds(rule) {
|
|
prior[s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: k}] = engine.PriorState{
|
|
BootstrapComplete: true,
|
|
Mode: engine.ModeEventDriven,
|
|
}
|
|
}
|
|
return prior
|
|
}
|
|
|
|
func eventCreate(bucket, key string, modTimeS, size int64, ts int64) *reader.Event {
|
|
return &reader.Event{
|
|
TsNs: ts,
|
|
Bucket: bucket,
|
|
Key: key,
|
|
NewEntry: &filer_pb.Entry{
|
|
Name: key,
|
|
Attributes: &filer_pb.FuseAttributes{
|
|
Mtime: modTimeS,
|
|
FileSize: uint64(size),
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestRouteNoSnapshotNoMatches(t *testing.T) {
|
|
if got := Route(nil, eventCreate("bk", "k", 0, 1, 1), time.Now()); got != nil {
|
|
t.Fatalf("nil snap should yield nil, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestRouteMissingBucketNoMatches(t *testing.T) {
|
|
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
|
|
snap := compileWith(rule, activatedPrior(rule))
|
|
ev := eventCreate("other-bucket", "k", 0, 1, 1)
|
|
if got := Route(snap, ev, time.Now()); got != nil {
|
|
t.Fatalf("foreign bucket should yield nil, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestRouteInactiveSkipped(t *testing.T) {
|
|
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
|
|
// PriorStates omitted => BootstrapComplete=false => action stays inactive.
|
|
snap := compileWith(rule, nil)
|
|
|
|
now := time.Now()
|
|
old := now.Add(-48 * time.Hour) // past 1-day expiration
|
|
ev := eventCreate("bk", "k", old.Unix(), 1, old.UnixNano())
|
|
|
|
if got := Route(snap, ev, now); got != nil {
|
|
t.Fatalf("inactive action should not match, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestRouteExpirationDaysFires(t *testing.T) {
|
|
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
|
|
snap := compileWith(rule, activatedPrior(rule))
|
|
|
|
now := time.Now()
|
|
old := now.Add(-48 * time.Hour) // past 1-day expiration
|
|
ev := eventCreate("bk", "k", old.Unix(), 1, old.UnixNano())
|
|
|
|
matches := Route(snap, ev, now)
|
|
if len(matches) != 1 {
|
|
t.Fatalf("expected 1 match, got %+v", matches)
|
|
}
|
|
m := matches[0]
|
|
if m.Bucket != "bk" || m.ObjectKey != "k" {
|
|
t.Fatalf("unexpected match shape: %+v", m)
|
|
}
|
|
if m.Result.Action != s3lifecycle.ActionDeleteObject {
|
|
t.Fatalf("action=%v, want DeleteObject", m.Result.Action)
|
|
}
|
|
if m.DueTime.Before(m.EventTs) {
|
|
t.Fatalf("DueTime < EventTs: %v < %v", m.DueTime, m.EventTs)
|
|
}
|
|
}
|
|
|
|
func TestRouteFreshObjectSchedulesInFuture(t *testing.T) {
|
|
// A fresh object DOES route — the action is scheduled for ModTime+N days.
|
|
// EvaluateAction is called at the scheduled dispatch time so the age
|
|
// gate passes; the dispatcher waits until DueTime to actually run.
|
|
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 7}
|
|
snap := compileWith(rule, activatedPrior(rule))
|
|
|
|
now := time.Now()
|
|
ev := eventCreate("bk", "k", now.Unix(), 1, now.UnixNano())
|
|
|
|
matches := Route(snap, ev, now)
|
|
if len(matches) != 1 {
|
|
t.Fatalf("expected 1 match (scheduled), got %v", matches)
|
|
}
|
|
if !matches[0].DueTime.After(now.Add(6 * 24 * time.Hour)) {
|
|
t.Fatalf("DueTime=%v, want ~7 days from now", matches[0].DueTime)
|
|
}
|
|
}
|
|
|
|
func TestRouteRespectsPrefixFilter(t *testing.T) {
|
|
rule := &s3lifecycle.Rule{
|
|
ID: "r", Status: s3lifecycle.StatusEnabled,
|
|
Prefix: "logs/", ExpirationDays: 1,
|
|
}
|
|
snap := compileWith(rule, activatedPrior(rule))
|
|
now := time.Now()
|
|
old := now.Add(-48 * time.Hour)
|
|
|
|
// Out of prefix: no match.
|
|
ev := eventCreate("bk", "data/file", old.Unix(), 1, old.UnixNano())
|
|
if got := Route(snap, ev, now); got != nil {
|
|
t.Fatalf("out-of-prefix should not match, got %v", got)
|
|
}
|
|
|
|
// In prefix: matches.
|
|
ev = eventCreate("bk", "logs/file", old.Unix(), 1, old.UnixNano())
|
|
if got := Route(snap, ev, now); len(got) != 1 {
|
|
t.Fatalf("in-prefix should match, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestRouteSkipsHardDelete(t *testing.T) {
|
|
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
|
|
snap := compileWith(rule, activatedPrior(rule))
|
|
now := time.Now()
|
|
old := now.Add(-48 * time.Hour)
|
|
// Hard delete: NewEntry is nil; OldEntry holds the gone object.
|
|
ev := &reader.Event{
|
|
TsNs: old.UnixNano(),
|
|
Bucket: "bk",
|
|
Key: "gone.txt",
|
|
OldEntry: &filer_pb.Entry{
|
|
Name: "gone.txt",
|
|
Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 1},
|
|
},
|
|
}
|
|
if got := Route(snap, ev, now); got != nil {
|
|
t.Fatalf("hard delete should not route, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestRouteSkipsMissingAttributes(t *testing.T) {
|
|
// Without Attributes there's no ModTime, and EvaluateAction would
|
|
// compute due against year-0001 and fire immediately. Skip the event.
|
|
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
|
|
snap := compileWith(rule, activatedPrior(rule))
|
|
ev := &reader.Event{
|
|
TsNs: time.Now().UnixNano(),
|
|
Bucket: "bk",
|
|
Key: "k",
|
|
NewEntry: &filer_pb.Entry{Name: "k"}, // no Attributes
|
|
}
|
|
if got := Route(snap, ev, time.Now()); got != nil {
|
|
t.Fatalf("missing-Attributes event should not route, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestRouteIdentityCapturedForNewEntry(t *testing.T) {
|
|
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
|
|
snap := compileWith(rule, activatedPrior(rule))
|
|
|
|
now := time.Now()
|
|
old := now.Add(-48 * time.Hour)
|
|
ev := eventCreate("bk", "k", old.Unix(), 42, old.UnixNano())
|
|
ev.NewEntry.Chunks = []*filer_pb.FileChunk{{FileId: "1,abc"}}
|
|
|
|
matches := Route(snap, ev, now)
|
|
if len(matches) != 1 {
|
|
t.Fatalf("expected 1 match, got %v", matches)
|
|
}
|
|
id := matches[0].Identity
|
|
if id == nil || id.Size != 42 || id.HeadFid != "1,abc" {
|
|
t.Fatalf("identity capture: %+v", id)
|
|
}
|
|
wantNs := old.Unix()*int64(1e9) + 0
|
|
if id.MtimeNs != wantNs {
|
|
t.Fatalf("MtimeNs=%d, want %d (Mtime*1e9)", id.MtimeNs, wantNs)
|
|
}
|
|
}
|