Files
seaweedfs/weed/s3api/s3lifecycle/dailyrun/run_stats.go
T
Chris LuandClaude Opus 5.5 fd4603d657 s3 lifecycle: count filer entries a walk reads but skips as progress (#12658)
* dailyrun: count entries read from the filer, skipped ones included

A versioned bucket's pre-pass and a resumed walk read entries without
emitting them, so a walk can read for hours with no visited entry.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3 lifecycle: count listed entries as progress work

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 11:12:33 -08:00

266 lines
5.7 KiB
Go

package dailyrun
import (
"context"
"strings"
"sync"
"sync/atomic"
"google.golang.org/grpc"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
)
// Bounds what a summary carries per pass; the counts stay exact.
const (
maxReportedBucketFailures = 10
maxReportedErrorLen = 256
)
const (
PhaseWalk = "walk"
PhaseReplay = "replay"
)
type bucketState uint8
const (
bucketSkipped bucketState = iota + 1
bucketWalked
bucketFailed
)
type BucketFailure struct {
Bucket string
Error string
}
// RunSummary is a copy of a RunStats at one point in time. A bucket walked
// more than once in a pass counts once, as failed if any walk failed.
type RunSummary struct {
Phase string
Bucket string
BucketsWalked int
BucketsFailed int
BucketsSkipped int
// The first maxReportedBucketFailures failed buckets.
BucketFailures []BucketFailure
// Every listed entry, versions and delete markers included.
EntriesVisited int64
// Distinct object keys among the listed entries.
ObjectsVisited int64
// Every entry read from the filer, including the ones a walk skips
// without visiting (versioned pre-pass, resume start, other shards).
EntriesListed int64
EventsScanned int64
// LifecycleDelete outcomes keyed by the dispatch_total outcome label.
// They count RPC results, not reclaimed space.
Outcomes map[string]int64
ShardErrors int
// Shards whose replay stopped on an unresolved delete; their due
// events wait for a later pass.
ShardsHalted int
// Shards whose walk-only rules waited for WalkerInterval.
WalksDeferred int
}
// RunStats collects one pass's work. Safe for concurrent use; a nil
// *RunStats ignores every call.
type RunStats struct {
entries atomic.Int64
objects atomic.Int64
listed atomic.Int64
events atomic.Int64
mu sync.Mutex
phase string
bucket string
buckets map[string]bucketState
failures []BucketFailure
outcomes map[string]int64
shardErrors int
shardsHalted int
deferred int
}
func (s *RunStats) startBucket(bucket string) {
if s == nil {
return
}
s.mu.Lock()
s.phase, s.bucket = PhaseWalk, bucket
s.mu.Unlock()
}
func (s *RunStats) finishBucket(bucket string, state bucketState, err error) {
if s == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
if s.buckets == nil {
s.buckets = map[string]bucketState{}
}
prev := s.buckets[bucket]
if state <= prev {
return
}
s.buckets[bucket] = state
if state == bucketFailed && len(s.failures) < maxReportedBucketFailures {
s.failures = append(s.failures, BucketFailure{Bucket: bucket, Error: truncateError(err)})
}
}
func (s *RunStats) startReplay() {
if s == nil {
return
}
s.mu.Lock()
s.phase, s.bucket = PhaseReplay, ""
s.mu.Unlock()
}
func (s *RunStats) visitEntry(newObject bool) {
if s == nil {
return
}
s.entries.Add(1)
if newObject {
s.objects.Add(1)
}
}
// CountListed returns client with its ListEntries results counted as
// EntriesListed.
func (s *RunStats) CountListed(client filer_pb.SeaweedFilerClient) filer_pb.SeaweedFilerClient {
if s == nil || client == nil {
return client
}
return &listCountingClient{SeaweedFilerClient: client, stats: s}
}
type listCountingClient struct {
filer_pb.SeaweedFilerClient
stats *RunStats
}
func (c *listCountingClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (filer_pb.SeaweedFiler_ListEntriesClient, error) {
stream, err := c.SeaweedFilerClient.ListEntries(ctx, in, opts...)
if err != nil {
return nil, err
}
return &listCountingStream{SeaweedFiler_ListEntriesClient: stream, stats: c.stats}, nil
}
type listCountingStream struct {
filer_pb.SeaweedFiler_ListEntriesClient
stats *RunStats
}
func (s *listCountingStream) Recv() (*filer_pb.ListEntriesResponse, error) {
resp, err := s.SeaweedFiler_ListEntriesClient.Recv()
if err == nil && resp.GetEntry() != nil {
s.stats.listed.Add(1)
}
return resp, err
}
func (s *RunStats) scanEvent() {
if s == nil {
return
}
s.events.Add(1)
}
func (s *RunStats) recordOutcome(outcome string) {
if s == nil {
return
}
s.mu.Lock()
if s.outcomes == nil {
s.outcomes = map[string]int64{}
}
s.outcomes[outcome]++
s.mu.Unlock()
}
func (s *RunStats) setShardErrors(n int) {
if s == nil {
return
}
s.mu.Lock()
s.shardErrors = n
s.mu.Unlock()
}
func (s *RunStats) shardHalted() {
if s == nil {
return
}
s.mu.Lock()
s.shardsHalted++
s.mu.Unlock()
}
func (s *RunStats) walkDeferred() {
if s == nil {
return
}
s.mu.Lock()
s.deferred++
s.mu.Unlock()
}
func (s *RunStats) Summary() RunSummary {
if s == nil {
return RunSummary{}
}
out := RunSummary{
EntriesVisited: s.entries.Load(),
ObjectsVisited: s.objects.Load(),
EntriesListed: s.listed.Load(),
EventsScanned: s.events.Load(),
}
s.mu.Lock()
defer s.mu.Unlock()
out.Phase, out.Bucket = s.phase, s.bucket
for _, state := range s.buckets {
switch state {
case bucketWalked:
out.BucketsWalked++
case bucketFailed:
out.BucketsFailed++
case bucketSkipped:
out.BucketsSkipped++
}
}
out.BucketFailures = append([]BucketFailure(nil), s.failures...)
if len(s.outcomes) > 0 {
out.Outcomes = make(map[string]int64, len(s.outcomes))
for k, v := range s.outcomes {
out.Outcomes[k] = v
}
}
out.ShardErrors = s.shardErrors
out.ShardsHalted = s.shardsHalted
out.WalksDeferred = s.deferred
return out
}
func countDispatch(s *RunStats, bucket, kind, outcome string) {
stats.S3LifecycleDispatchCounter.WithLabelValues(bucket, kind, outcome).Inc()
s.recordOutcome(outcome)
}
func truncateError(err error) string {
if err == nil {
return ""
}
msg := err.Error()
if len(msg) <= maxReportedErrorLen {
return msg
}
return strings.ToValidUTF8(msg[:maxReportedErrorLen], "") + "..."
}