Files
seaweedfs/weed/s3api/s3lifecycle/dailyrun/walker_recovery_test.go
T
Chris Lu 37e505b8fd refactor(s3/lifecycle): one meta-log subscription per dailyrun.Run pass (#9481)
* refactor(s3/lifecycle): one meta-log subscription per dailyrun.Run pass

Per-shard Reader subscriptions multiplied filer load by len(cfg.Shards)
even though the same gRPC stream could serve every shard in a worker
process. Replace with one SubscribeMetadata stream covering all shards
in cfg.Shards: the Reader's ShardPredicate accepts the shard set, and
a fan-out goroutine routes events to per-shard channels by ev.ShardID.

drainShardEvents now reads from a passed-in channel; shards whose
persisted cursor is fresher than the global floor (runNow - maxTTL)
filter ev.TsNs <= startTsNs locally. The fan-out cancels the reader
when the first ev.TsNs > runNow arrives — meta-log order means the
rest of the stream is past the pass boundary too.

cfg.Workers no longer gates shard concurrency: with the shared
subscription, every shard goroutine must be live to drain its channel,
or the fan-out stalls. The field is retained for back-compat and
ignored. Dispatch throttling still goes through cfg.Limiter.

Filer load: 16x -> 1x SubscribeMetadata streams per pass.

* fix(s3/lifecycle): shared subscription floor is min(per-shard cursor)

The shared subscription used runNow - maxTTL as its starting TsNs, but
that's the cold-start floor. For shards whose persisted cursor sits
below the floor — exactly the case a rule with TTL == maxTTL produces,
where a pending event's PUT TsNs ends up at runNow - maxTTL — events
that the per-shard drain still needs are filtered out before the
Reader even forwards them.

Same regression I fixed in 6796ab6db for the per-shard subscription;
now applied at the shared level. computeGlobalStartTsNs loads every
shard's cursor and picks the minimum, falling back to the cold-start
floor only for shards with no persisted cursor.
2026-05-13 02:13:11 -07:00

144 lines
4.5 KiB
Go

package dailyrun
import (
"context"
"errors"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// memPersister is a minimal in-memory CursorPersister. Phase 4b tests
// drive runShard directly; the recovery-branch path returns before
// drainShardEvents so the heavier filer/client/lister fakes aren't
// needed here.
type memPersister struct {
store map[int]Cursor
}
func newMemPersister() *memPersister { return &memPersister{store: map[int]Cursor{}} }
func (p *memPersister) Load(_ context.Context, shardID int) (Cursor, bool, error) {
c, ok := p.store[shardID]
return c, ok, nil
}
func (p *memPersister) Save(_ context.Context, shardID int, c Cursor) error {
p.store[shardID] = c
return nil
}
func snapshotWithRule(t *testing.T, days int) *engine.Snapshot {
t.Helper()
e := engine.New()
e.Compile([]engine.CompileInput{
{Bucket: "b1", Rules: []*s3lifecycle.Rule{
{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: days},
}},
}, engine.CompileOptions{})
snap := e.Snapshot()
for _, a := range snap.AllActions() {
snap.MarkActive(a.Key)
}
return snap
}
func TestRunShard_WalkerInvokedOnRecoveryBranch(t *testing.T) {
snap := snapshotWithRule(t, 30)
p := newMemPersister()
// Seed a persisted cursor whose RuleSetHash differs from snap's, so
// the rule-change branch fires.
var stale [32]byte
for i := range stale {
stale[i] = 0xAA
}
require.NoError(t, p.Save(context.Background(), 3, Cursor{TsNs: 1234, RuleSetHash: stale}))
var gotView *engine.Snapshot
var gotShard int
calls := 0
cfg := Config{
Persister: p,
Walker: func(_ context.Context, view *engine.Snapshot, shardID int) error {
calls++
gotView = view
gotShard = shardID
return nil
},
}
runNow := time.Unix(1_700_000_000, 0).UTC()
require.NoError(t, runShard(context.Background(), cfg, snap, runNow, 3, nil))
assert.Equal(t, 1, calls, "walker must fire exactly once on recovery")
require.NotNil(t, gotView, "walker received the RecoveryView")
assert.Equal(t, 3, gotShard)
// Cursor rewound to runNow - maxTTL with the new hashes persisted.
got, ok, err := p.Load(context.Background(), 3)
require.NoError(t, err)
require.True(t, ok)
expectedFloor := runNow.Add(-engine.MaxEffectiveTTL(snap)).UnixNano()
assert.Equal(t, expectedFloor, got.TsNs, "cursor must rewind to runNow - maxTTL after recovery walk")
assert.NotEqual(t, stale, got.RuleSetHash, "stale hash must be replaced")
}
func TestRunShard_NilWalkerOnRecoveryIsNoop(t *testing.T) {
// Phase 4a behavior: a nil Walker must not crash and must still
// rewind the cursor.
snap := snapshotWithRule(t, 30)
p := newMemPersister()
var stale [32]byte
for i := range stale {
stale[i] = 0xBB
}
require.NoError(t, p.Save(context.Background(), 0, Cursor{TsNs: 9999, RuleSetHash: stale}))
cfg := Config{Persister: p} // Walker nil
runNow := time.Unix(1_700_000_000, 0).UTC()
require.NoError(t, runShard(context.Background(), cfg, snap, runNow, 0, nil))
got, ok, err := p.Load(context.Background(), 0)
require.NoError(t, err)
require.True(t, ok)
assert.Equal(t, runNow.Add(-engine.MaxEffectiveTTL(snap)).UnixNano(), got.TsNs)
}
func TestRunShard_WalkerErrorPropagates(t *testing.T) {
// A walker failure must surface as the runShard error so the daily
// run treats the shard as interrupted, and must NOT advance the
// cursor (the seeded cursor stays).
snap := snapshotWithRule(t, 30)
p := newMemPersister()
var stale [32]byte
for i := range stale {
stale[i] = 0xCC
}
require.NoError(t, p.Save(context.Background(), 7, Cursor{TsNs: 42, RuleSetHash: stale}))
cfg := Config{
Persister: p,
Walker: func(_ context.Context, _ *engine.Snapshot, _ int) error {
return errors.New("walker boom")
},
}
runNow := time.Unix(1_700_000_000, 0).UTC()
err := runShard(context.Background(), cfg, snap, runNow, 7, nil)
require.Error(t, err)
assert.Contains(t, err.Error(), "walker boom")
// Cursor untouched.
got, _, _ := p.Load(context.Background(), 7)
assert.Equal(t, int64(42), got.TsNs, "walker failure must leave cursor unchanged")
assert.Equal(t, stale, got.RuleSetHash)
}
// The "matching cursor doesn't invoke walker" case is implicit: the
// walker call is inside the rule-change / partition-flip `if` and
// can't be reached otherwise. Exercising it end-to-end requires the
// full filer + lister + client harness; covered by the integration
// tests once Phase 4b is wired into the handler.