mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-31 17:30:42 +01:00
Chris Lu
443ae1cd83
* s3 lifecycle: share the daily bucket walk across shards The daily pass invoked the walker once per shard, and every shard's walk listed the full bucket tree while keeping only its own 1/16 of the entries — 16 full listings per pass by default (#11791). Run now groups the shards by the walk branch they'll take (recovery vs steady, using the same cursor data and hash checks runShard applies) and starts one shared walk per group, so a pass lists each bucket at most twice. WalkerFunc and WalkBuckets take a shard set instead of a single shard. runShard keeps its own decision and waits on its group's walk; a shard whose freshly-loaded cursor disagrees with the plan falls back to a per-shard walk, so mid-pass cursor rewrites can't under-cover a shard. * s3 lifecycle: skip the .versions/ pre-pass on non-versioned buckets walkBucketTree listed every directory twice — once looking for .versions/ dirs, once for regular entries. On a bucket that never had versioning enabled the first pass can only find coincidentally-named user folders, so the extra listing is pure cost (#11791). FilerListFunc now takes the walk view and consults Snapshot.BucketVersioned; non-versioned buckets skip pass 1 entirely. A *-suffixed .versions user folder on such a bucket is recursed into as a regular directory so real objects inside still surface; a nil view keeps the two-pass behavior. * s3 lifecycle: prune walk directories to active rule prefixes The daily walk listed every directory under a bucket even when no rule prefix could reach it. Collect the active rule prefixes of each bucket from the walk view (new Snapshot.BucketPrefixes) and skip recursing into directories whose keys can never match; a bucket with no active actions is skipped entirely, saving the listing. .uploads stays in scope regardless of prefixes because AbortMPU rules match DestKey, and .versions expansion is pruned only conservatively (the logical key is a string-prefix of the dir key). * s3 lifecycle: join planned walks before Run returns planWalks starts a shared walk goroutine per group, but a shard whose authoritative cursor load then fails exits runShard without waiting on it. Run returned while the walk still listed or dispatched deletes against connections the caller closes on return. Join every planned walk after the shard waitgroup. * s3 lifecycle: treat object-locked buckets as versioned in the walk Object Lock requires versioning and every writer sets both extended attributes, but IsBucketVersioned only read ExtVersioningKey. A locked bucket missing that attribute would have its .versions dirs walked as ordinary folders, hiding version siblings from lifecycle. Classify ExtObjectLockEnabledKey as versioned too.
81 lines
2.6 KiB
Go
81 lines
2.6 KiB
Go
package dailyrun
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
|
|
)
|
|
|
|
// WalkBuckets runs bootstrap.Walk for each bucket using a per-bucket
|
|
// ListFunc wrapped to drop entries whose ShardID isn't in shards.
|
|
// Returns the first bucket error; remaining buckets log and continue
|
|
// so one bucket's filer error doesn't kill the whole walk.
|
|
//
|
|
// Composable shape: callers (the worker handler) supply a real
|
|
// filer-backed ListFunc and a real Dispatcher (WalkerDispatcher).
|
|
// Tests pass bootstrap.EntryCallback and a stub Dispatcher.
|
|
func WalkBuckets(ctx context.Context, view *engine.Snapshot, shards []int, buckets []string, list bootstrap.ListFunc, dispatch bootstrap.Dispatcher) error {
|
|
if view == nil {
|
|
return errors.New("WalkBuckets: nil view")
|
|
}
|
|
if list == nil {
|
|
return errors.New("WalkBuckets: nil list")
|
|
}
|
|
if dispatch == nil {
|
|
return errors.New("WalkBuckets: nil dispatch")
|
|
}
|
|
shardSet := make(map[int]bool, len(shards))
|
|
for _, sh := range shards {
|
|
shardSet[sh] = true
|
|
}
|
|
filtered := perShardListFunc(list, shardSet)
|
|
var firstErr error
|
|
for _, b := range buckets {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
if _, err := bootstrap.Walk(ctx, view, b, filtered, dispatch, bootstrap.WalkOptions{}); err != nil {
|
|
if firstErr == nil {
|
|
firstErr = fmt.Errorf("walk %s: %w", b, err)
|
|
} else {
|
|
glog.V(1).Infof("walker: additional bucket %s: %v", b, err)
|
|
}
|
|
}
|
|
}
|
|
return firstErr
|
|
}
|
|
|
|
// perShardListFunc wraps an inner ListFunc so only entries whose
|
|
// logical-key shard is in shardSet reach the callback. The walker
|
|
// emits one entry per logical object — versioned siblings share the
|
|
// same logical key so they're either all in-shard or all out — and
|
|
// MPU init records carry the user's destination in DestKey, so the
|
|
// shard predicate must use DestKey there to match what the dispatcher
|
|
// will send.
|
|
func perShardListFunc(inner bootstrap.ListFunc, shardSet map[int]bool) bootstrap.ListFunc {
|
|
return func(ctx context.Context, bucket, start string, cb func(*bootstrap.Entry) error) error {
|
|
return inner(ctx, bucket, start, func(e *bootstrap.Entry) error {
|
|
if e == nil {
|
|
return nil
|
|
}
|
|
if !shardSet[entryShardID(bucket, e)] {
|
|
return nil
|
|
}
|
|
return cb(e)
|
|
})
|
|
}
|
|
}
|
|
|
|
func entryShardID(bucket string, e *bootstrap.Entry) int {
|
|
key := e.Path
|
|
if e.IsMPUInit && e.DestKey != "" {
|
|
key = e.DestKey
|
|
}
|
|
return s3lifecycle.ShardID(bucket, key)
|
|
}
|