mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 16:56:04 +01:00
* s3 lifecycle: collect per-pass bucket results and work counters WalkBuckets kept only the first bucket error and logged the rest. A RunStats shared by the walker, the walker dispatcher and the replay drain now records every bucket's walk result (walked, failed, skipped on cancellation), entry and object visits, replay events, and LifecycleDelete outcomes under the dispatch_total outcome labels. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: persist the last fully successful pass Shard cursors and walk timestamps advance on partial passes as well, so neither says when lifecycle last covered every policy. Keep that in its own record next to the cursors, plus a worker gauge for it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: report each pass's result in the job record The job ended with a fixed success summary, or with only the first error, and malformed lifecycle policies were V(1) logs. The completion now carries the pass status (complete, partial, failed, canceled), bucket totals and failures, invalid policies, entry, object and event counts, LifecycleDelete outcomes, the last phase and bucket, the elapsed time and the last full pass, on failed passes as well. Only a complete pass advances the full-pass record; a malformed policy makes the pass partial. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * admin: keep a failed job's result in its record Completions with Success=false dropped Result.Summary and Result.OutputValues, so a handler could not report what a failed run had done, which buckets failed, or how far it got. Keep them; the error message still drives the job's message and run record. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * grafana: chart the time since the last full lifecycle pass Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: a replay halted on an unresolved delete is a partial pass BLOCKED, RETRY_LATER and exhausted transport retries stop a shard's replay without an error, leaving its due events for a later pass. Count those shards and report the pass as partial, so it no longer advances the full-pass record. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: a pass with throttled walks is not a full pass WalkerInterval can skip the walk for walk-only rules (ExpirationDate, ExpiredDeleteMarker, NewerNoncurrent) with no error. Count those shards and keep the full-pass record for passes that evaluated every rule; the job reports full_pass and walks_deferred. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: keep the saved full-pass time when the save fails Publishing a timestamp that never reached the filer let the gauge move back on the next job's reload. Keep the previous value and report the save error in the job result. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * admin: drop a failed attempt's summary when the retry reports none Retries reuse the job record, and failed completions now keep their result summary, so a retry that completes without a Result showed the failed attempt's summary, and a successful one used it as its message. Clear the summary when a completion carries no Result, as the output values already are. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
101 lines
3.2 KiB
Go
101 lines
3.2 KiB
Go
package dailyrun
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
|
|
)
|
|
|
|
// WalkBuckets runs bootstrap.Walk for each bucket using a per-bucket
|
|
// ListFunc wrapped to drop entries whose ShardID isn't in shards.
|
|
// Returns the first bucket error; remaining buckets log and continue
|
|
// so one bucket's filer error doesn't kill the whole walk. Every
|
|
// bucket's result goes to runStats when it is non-nil.
|
|
//
|
|
// Composable shape: callers (the worker handler) supply a real
|
|
// filer-backed ListFunc and a real Dispatcher (WalkerDispatcher).
|
|
// Tests pass bootstrap.EntryCallback and a stub Dispatcher.
|
|
func WalkBuckets(ctx context.Context, view *engine.Snapshot, shards []int, buckets []string, list bootstrap.ListFunc, dispatch bootstrap.Dispatcher, runStats *RunStats) error {
|
|
if view == nil {
|
|
return errors.New("WalkBuckets: nil view")
|
|
}
|
|
if list == nil {
|
|
return errors.New("WalkBuckets: nil list")
|
|
}
|
|
if dispatch == nil {
|
|
return errors.New("WalkBuckets: nil dispatch")
|
|
}
|
|
shardSet := make(map[int]bool, len(shards))
|
|
for _, sh := range shards {
|
|
shardSet[sh] = true
|
|
}
|
|
filtered := perShardListFunc(list, shardSet, runStats)
|
|
var firstErr error
|
|
for i, b := range buckets {
|
|
if err := ctx.Err(); err != nil {
|
|
for _, rest := range buckets[i:] {
|
|
runStats.finishBucket(rest, bucketSkipped, nil)
|
|
}
|
|
return err
|
|
}
|
|
runStats.startBucket(b)
|
|
_, err := bootstrap.Walk(ctx, view, b, filtered, dispatch, bootstrap.WalkOptions{})
|
|
switch {
|
|
case err == nil:
|
|
runStats.finishBucket(b, bucketWalked, nil)
|
|
case ctx.Err() != nil:
|
|
runStats.finishBucket(b, bucketSkipped, nil)
|
|
default:
|
|
runStats.finishBucket(b, bucketFailed, err)
|
|
}
|
|
if err != nil {
|
|
if firstErr == nil {
|
|
firstErr = fmt.Errorf("walk %s: %w", b, err)
|
|
} else {
|
|
glog.V(1).Infof("walker: additional bucket %s: %v", b, err)
|
|
}
|
|
}
|
|
}
|
|
return firstErr
|
|
}
|
|
|
|
// perShardListFunc wraps an inner ListFunc so only entries whose
|
|
// logical-key shard is in shardSet reach the callback. The walker
|
|
// emits one entry per logical object — versioned siblings share the
|
|
// same logical key so they're either all in-shard or all out — and
|
|
// MPU init records carry the user's destination in DestKey, so the
|
|
// shard predicate must use DestKey there to match what the dispatcher
|
|
// will send.
|
|
func perShardListFunc(inner bootstrap.ListFunc, shardSet map[int]bool, runStats *RunStats) bootstrap.ListFunc {
|
|
return func(ctx context.Context, bucket, start string, cb func(*bootstrap.Entry) error) error {
|
|
// Listings emit every version of a key back to back.
|
|
lastPath := ""
|
|
return inner(ctx, bucket, start, func(e *bootstrap.Entry) error {
|
|
if e == nil {
|
|
return nil
|
|
}
|
|
if !shardSet[entryShardID(bucket, e)] {
|
|
return nil
|
|
}
|
|
if !e.IsDirectory || e.IsMPUInit {
|
|
runStats.visitEntry(!e.IsMPUInit && e.Path != lastPath)
|
|
lastPath = e.Path
|
|
}
|
|
return cb(e)
|
|
})
|
|
}
|
|
}
|
|
|
|
func entryShardID(bucket string, e *bootstrap.Entry) int {
|
|
key := e.Path
|
|
if e.IsMPUInit && e.DestKey != "" {
|
|
key = e.DestKey
|
|
}
|
|
return s3lifecycle.ShardID(bucket, key)
|
|
}
|