mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 17:55:26 +01:00
* s3 lifecycle: collect per-pass bucket results and work counters WalkBuckets kept only the first bucket error and logged the rest. A RunStats shared by the walker, the walker dispatcher and the replay drain now records every bucket's walk result (walked, failed, skipped on cancellation), entry and object visits, replay events, and LifecycleDelete outcomes under the dispatch_total outcome labels. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: persist the last fully successful pass Shard cursors and walk timestamps advance on partial passes as well, so neither says when lifecycle last covered every policy. Keep that in its own record next to the cursors, plus a worker gauge for it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: report each pass's result in the job record The job ended with a fixed success summary, or with only the first error, and malformed lifecycle policies were V(1) logs. The completion now carries the pass status (complete, partial, failed, canceled), bucket totals and failures, invalid policies, entry, object and event counts, LifecycleDelete outcomes, the last phase and bucket, the elapsed time and the last full pass, on failed passes as well. Only a complete pass advances the full-pass record; a malformed policy makes the pass partial. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * admin: keep a failed job's result in its record Completions with Success=false dropped Result.Summary and Result.OutputValues, so a handler could not report what a failed run had done, which buckets failed, or how far it got. Keep them; the error message still drives the job's message and run record. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * grafana: chart the time since the last full lifecycle pass Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: a replay halted on an unresolved delete is a partial pass BLOCKED, RETRY_LATER and exhausted transport retries stop a shard's replay without an error, leaving its due events for a later pass. Count those shards and report the pass as partial, so it no longer advances the full-pass record. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: a pass with throttled walks is not a full pass WalkerInterval can skip the walk for walk-only rules (ExpirationDate, ExpiredDeleteMarker, NewerNoncurrent) with no error. Count those shards and keep the full-pass record for passes that evaluated every rule; the job reports full_pass and walks_deferred. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: keep the saved full-pass time when the save fails Publishing a timestamp that never reached the filer let the gauge move back on the next job's reload. Keep the previous value and report the save error in the job result. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * admin: drop a failed attempt's summary when the retry reports none Retries reuse the job record, and failed completions now keep their result summary, so a retry that completes without a Result showed the failed attempt's summary, and a successful one used it as its message. Clear the summary when a completion carries no Result, as the output values already are. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
176 lines
5.8 KiB
Go
176 lines
5.8 KiB
Go
package dailyrun
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sort"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// recordingDispatcher captures every (action, entry) pair the walker
|
|
// emits so tests can assert dispatch order and shard filtering.
|
|
type recordingDispatcher struct {
|
|
calls []recordedDispatch
|
|
err error
|
|
}
|
|
|
|
type recordedDispatch struct {
|
|
bucket string
|
|
path string
|
|
}
|
|
|
|
func (d *recordingDispatcher) Delete(_ context.Context, action *engine.CompiledAction, entry *bootstrap.Entry) error {
|
|
d.calls = append(d.calls, recordedDispatch{bucket: action.Bucket, path: entry.Path})
|
|
return d.err
|
|
}
|
|
|
|
// findShardForPath returns the shard ID for an entry with given path
|
|
// in given bucket. Helper for tests that want to force entries onto
|
|
// a known shard.
|
|
func findShardForPath(bucket, path string) int {
|
|
return s3lifecycle.ShardID(bucket, path)
|
|
}
|
|
|
|
// fixedShardEntries returns a slice of bootstrap.Entries whose paths
|
|
// all live in the same shard. Used so a per-shard filter test has
|
|
// deterministic expectations.
|
|
func fixedShardEntries(t *testing.T, bucket string, shardID int, count int) []*bootstrap.Entry {
|
|
t.Helper()
|
|
var out []*bootstrap.Entry
|
|
for i := 0; tries(i, count); i++ {
|
|
path := "obj-" + intToStr(i)
|
|
if findShardForPath(bucket, path) == shardID {
|
|
out = append(out, &bootstrap.Entry{
|
|
Path: path,
|
|
// Old enough to expire under a 7-day ExpirationDays rule.
|
|
ModTime: time.Now().Add(-90 * 24 * time.Hour),
|
|
Size: 1,
|
|
IsLatest: true,
|
|
})
|
|
if len(out) >= count {
|
|
return out
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("could not find %d entries in shard %d for bucket %s after 4096 attempts", count, shardID, bucket)
|
|
return nil
|
|
}
|
|
|
|
func intToStr(i int) string {
|
|
const hex = "0123456789abcdef"
|
|
if i == 0 {
|
|
return "0"
|
|
}
|
|
var out []byte
|
|
for n := i; n > 0; n /= 16 {
|
|
out = append([]byte{hex[n%16]}, out...)
|
|
}
|
|
return string(out)
|
|
}
|
|
|
|
func tries(i, count int) bool { return i < count*256+256 }
|
|
|
|
func snapshotForBucketRule(t *testing.T, bucket string, days int) *engine.Snapshot {
|
|
t.Helper()
|
|
e := engine.New()
|
|
e.Compile([]engine.CompileInput{
|
|
{Bucket: bucket, Rules: []*s3lifecycle.Rule{
|
|
{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: days},
|
|
}},
|
|
}, engine.CompileOptions{})
|
|
snap := e.Snapshot()
|
|
for _, a := range snap.AllActions() {
|
|
snap.MarkActive(a.Key)
|
|
}
|
|
return snap
|
|
}
|
|
|
|
func TestWalkBuckets_DispatchesOnlyShardMatchingEntries(t *testing.T) {
|
|
bucket := "b1"
|
|
const targetShard = 0
|
|
// Build entries: half in shard 0, half elsewhere. The walker's
|
|
// per-shard filter must drop the non-matching ones.
|
|
inShard := fixedShardEntries(t, bucket, targetShard, 3)
|
|
otherShard := fixedShardEntries(t, bucket, (targetShard+1)%16, 3)
|
|
all := append([]*bootstrap.Entry{}, inShard...)
|
|
all = append(all, otherShard...)
|
|
|
|
snap := snapshotForBucketRule(t, bucket, 7) // 30-day-old objects expire
|
|
d := &recordingDispatcher{}
|
|
err := WalkBuckets(context.Background(), snap, []int{targetShard}, []string{bucket},
|
|
bootstrap.EntryCallback(all), d, nil)
|
|
require.NoError(t, err)
|
|
|
|
gotPaths := make([]string, 0, len(d.calls))
|
|
for _, c := range d.calls {
|
|
gotPaths = append(gotPaths, c.path)
|
|
assert.Equal(t, bucket, c.bucket)
|
|
}
|
|
sort.Strings(gotPaths)
|
|
|
|
wantPaths := make([]string, 0, len(inShard))
|
|
for _, e := range inShard {
|
|
wantPaths = append(wantPaths, e.Path)
|
|
}
|
|
sort.Strings(wantPaths)
|
|
assert.Equal(t, wantPaths, gotPaths, "walker must dispatch exactly the in-shard entries")
|
|
}
|
|
|
|
func TestWalkBuckets_NilGuards(t *testing.T) {
|
|
require.Error(t, WalkBuckets(context.Background(), nil, []int{0}, nil, bootstrap.EntryCallback(nil), &recordingDispatcher{}, nil))
|
|
require.Error(t, WalkBuckets(context.Background(), snapshotForBucketRule(t, "b", 7), []int{0}, nil, nil, &recordingDispatcher{}, nil))
|
|
require.Error(t, WalkBuckets(context.Background(), snapshotForBucketRule(t, "b", 7), []int{0}, nil, bootstrap.EntryCallback(nil), nil, nil))
|
|
}
|
|
|
|
func TestWalkBuckets_OneBucketErrorDoesNotStopOthers(t *testing.T) {
|
|
// Two buckets, first ListFunc errors. WalkBuckets should still
|
|
// process the second bucket but return the first bucket's error.
|
|
listErr := errors.New("filer flake")
|
|
list := func(_ context.Context, bucket, _ string, _ func(*bootstrap.Entry) error) error {
|
|
if bucket == "bad" {
|
|
return listErr
|
|
}
|
|
return nil
|
|
}
|
|
|
|
snap := snapshotForBucketRule(t, "good", 7)
|
|
err := WalkBuckets(context.Background(), snap, []int{0}, []string{"bad", "good"}, list, &recordingDispatcher{}, nil)
|
|
require.Error(t, err)
|
|
assert.ErrorIs(t, err, listErr)
|
|
}
|
|
|
|
func TestWalkBuckets_HonorsContextCancellationBetweenBuckets(t *testing.T) {
|
|
// Pre-cancel ctx before invocation. Even an empty bucket list
|
|
// must surface ctx.Err early so a long walk in progress can
|
|
// short-circuit.
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
err := WalkBuckets(ctx, snapshotForBucketRule(t, "b", 7), []int{0}, []string{"b"},
|
|
bootstrap.EntryCallback(nil), &recordingDispatcher{}, nil)
|
|
assert.ErrorIs(t, err, context.Canceled)
|
|
}
|
|
|
|
func TestEntryShardID_MPUUsesDestKey(t *testing.T) {
|
|
bucket := "bkt"
|
|
// Force a (Path, DestKey) pair that maps to different shards
|
|
// so we can prove the filter picks the right one.
|
|
for i := 0; i < 4096; i++ {
|
|
path := ".uploads/" + intToStr(i)
|
|
destKey := "user/" + intToStr(i)
|
|
if s3lifecycle.ShardID(bucket, path) != s3lifecycle.ShardID(bucket, destKey) {
|
|
e := &bootstrap.Entry{Path: path, DestKey: destKey, IsMPUInit: true}
|
|
assert.Equal(t, s3lifecycle.ShardID(bucket, destKey), entryShardID(bucket, e),
|
|
"MPU init must use DestKey for shard, not the .uploads/<id> path")
|
|
return
|
|
}
|
|
}
|
|
t.Fatal("could not find an MPU path/destkey pair with diverging shards in 4096 attempts")
|
|
}
|