diff --git a/weed/s3api/s3lifecycle/bootstrap/walker.go b/weed/s3api/s3lifecycle/bootstrap/walker.go index 845a459d5a..59325f410f 100644 --- a/weed/s3api/s3lifecycle/bootstrap/walker.go +++ b/weed/s3api/s3lifecycle/bootstrap/walker.go @@ -52,10 +52,15 @@ type Entry struct { // Identity is the listed entry's fingerprint; the server refuses the // delete when the live entry no longer matches it. Identity *router.EntryIdentity + + // Resume is the ListFunc start value that continues the walk after + // this entry. Empty keeps the previous position, as for every + // version of an object but its last. + Resume string } -// ListFunc must skip entries with Path <= start so kill-resume picks up -// where the previous run stopped. +// ListFunc lists the bucket, starting after start: "" or the Resume of +// an entry an earlier walk finished. type ListFunc func(ctx context.Context, bucket, start string, cb func(*Entry) error) error // Dispatcher executes one (action, entry) verdict. An error halts the walk; @@ -64,8 +69,8 @@ type Dispatcher interface { Delete(ctx context.Context, action *engine.CompiledAction, entry *Entry) error } -// Checkpoint is the resume state. Caller persists it under -// /etc/s3/lifecycle//_bootstrap. +// Checkpoint is the resume state: LastScannedPath is the Resume of the +// last finished entry, to pass back as WalkOptions.Resume. type Checkpoint struct { LastScannedPath string Completed bool @@ -97,14 +102,14 @@ func Walk(ctx context.Context, snap *engine.Snapshot, bucket string, list ListFu } // MPU init directories at .uploads/ are the one directory // shape lifecycle cares about; everything else stays out. - if entry.IsDirectory && !entry.IsMPUInit { - cp.LastScannedPath = entry.Path - return nil + if !entry.IsDirectory || entry.IsMPUInit { + if err := walkEntry(ctx, snap, bucket, entry, dispatch, now, &info); err != nil { + return err + } } - if err := walkEntry(ctx, snap, bucket, entry, dispatch, now, &info); err != nil { - return err + if entry.Resume != "" { + cp.LastScannedPath = entry.Resume } - cp.LastScannedPath = entry.Path return nil }) if err != nil { @@ -184,13 +189,17 @@ func walkEntry(ctx context.Context, snap *engine.Snapshot, bucket string, entry return nil } -// EntryCallback wraps an in-memory slice as a ListFunc; useful for tests. +// EntryCallback wraps an in-memory slice as a ListFunc that resumes by +// Path; useful for tests. func EntryCallback(entries []*Entry) ListFunc { return func(ctx context.Context, bucket, start string, cb func(*Entry) error) error { for _, e := range entries { if start != "" && e.Path <= start { continue } + if e != nil && e.Resume == "" { + e.Resume = e.Path + } if err := cb(e); err != nil { return err } diff --git a/weed/s3api/s3lifecycle/bootstrap/walker_test.go b/weed/s3api/s3lifecycle/bootstrap/walker_test.go index 113d20abeb..50adb9cfc1 100644 --- a/weed/s3api/s3lifecycle/bootstrap/walker_test.go +++ b/weed/s3api/s3lifecycle/bootstrap/walker_test.go @@ -429,3 +429,39 @@ func TestWalk_MPUInitDoesNotFireNoncurrent(t *testing.T) { t.Fatalf("kind=%v, want AbortMPU", rec.calls[0].kind) } } + +type failVersion string + +func (f failVersion) Delete(_ context.Context, _ *engine.CompiledAction, entry *Entry) error { + if entry.VersionID == string(f) { + return errors.New("dispatch boom") + } + return nil +} + +func TestWalk_CheckpointStaysBeforeUnfinishedVersionGroup(t *testing.T) { + rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1} + snap := compileEvDriven(t, "bk", rule) + mod := mustTime(t, "2024-01-01T00:00:00Z") + list := func(_ context.Context, _, _ string, cb func(*Entry) error) error { + for _, e := range []*Entry{ + {Path: "a", IsLatest: true, ModTime: mod, Resume: "after-a"}, + {Path: "obj", VersionID: "v2", IsLatest: true, ModTime: mod}, + {Path: "obj", VersionID: "v1", ModTime: mod, Resume: "after-obj"}, + } { + if err := cb(e); err != nil { + return err + } + } + return nil + } + + cp, err := Walk(context.Background(), snap, "bk", list, failVersion("v2"), WalkOptions{Now: mod.AddDate(0, 0, 10)}) + if err == nil || cp.LastScannedPath != "after-a" { + t.Fatalf("failure inside the group: got cp=%+v err=%v, want after-a and an error", cp, err) + } + cp, err = Walk(context.Background(), snap, "bk", list, failVersion("none"), WalkOptions{Now: mod.AddDate(0, 0, 10)}) + if err != nil || !cp.Completed || cp.LastScannedPath != "after-obj" { + t.Fatalf("finished walk: got cp=%+v err=%v, want after-obj", cp, err) + } +} diff --git a/weed/s3api/s3lifecycle/dailyrun/filer_list_func.go b/weed/s3api/s3lifecycle/dailyrun/filer_list_func.go index f646564ecf..cf1e713dc6 100644 --- a/weed/s3api/s3lifecycle/dailyrun/filer_list_func.go +++ b/weed/s3api/s3lifecycle/dailyrun/filer_list_func.go @@ -8,6 +8,7 @@ import ( "sync/atomic" "time" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" @@ -39,6 +40,13 @@ func init() { listPageSize.Store(1024) } // active rule prefix can reach are pruned rather than listed. A nil // view assumes the worst (versioned, unpruned), preserving the // two-pass behavior. +// +// start and Entry.Resume are walk positions: one ":" frame +// per directory level, joined by "/", naming the last finished entry. +// They follow the traversal, which is not name order: at every level +// pass 1 (.versions groups) precedes pass 2, and a directory's children +// come before a sibling file such as "dir.txt". A version group is one +// unit, so a resume repeats it until all of its versions finished. func FilerListFunc(client filer_pb.SeaweedFilerClient, bucketsPath string, view *engine.Snapshot) bootstrap.ListFunc { return func(ctx context.Context, bucket, start string, cb func(*bootstrap.Entry) error) error { if client == nil { @@ -53,10 +61,47 @@ func FilerListFunc(client filer_pb.SeaweedFilerClient, bucketsPath string, view return nil // no active rule can match anything in this bucket } } - return walkBucketTree(ctx, client, root, root, start, versioned, prefixes, cb) + at, err := parseWalkPosition(start) + if err != nil { + glog.Warningf("lifecycle walk %s: %v; walking the whole bucket", bucket, err) + } + return walkBucketTree(ctx, client, root, root, "", at, versioned, prefixes, cb) } } +const ( + passVersions byte = '1' + passEntries byte = '2' +) + +type walkFrame struct { + pass byte + name string +} + +func walkPosition(parent string, pass byte, name string) string { + frame := string(pass) + ":" + name + if parent == "" { + return frame + } + return parent + "/" + frame +} + +func parseWalkPosition(pos string) ([]walkFrame, error) { + if pos == "" { + return nil, nil + } + parts := strings.Split(pos, "/") + frames := make([]walkFrame, len(parts)) + for i, p := range parts { + if len(p) < 3 || p[1] != ':' || (p[0] != passVersions && p[0] != passEntries) { + return nil, fmt.Errorf("bad walk position %q", pos) + } + frames[i] = walkFrame{pass: p[0], name: p[2:]} + } + return frames, nil +} + // walkBucketTree recurses through dir in two passes per level. Pass 1 // expands `.versions/` dirs (populating skipBare with the bare null- // version keys that pass 2 must suppress) and only runs when the bucket @@ -67,12 +112,27 @@ func FilerListFunc(client filer_pb.SeaweedFilerClient, bucketsPath string, view // `.versions/` dirs must be processed before their bare sibling so the // walk sees whether the bare entry is the null version of the same // logical object; hence the two passes on versioned buckets. -func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir, bucketRoot, start string, versioned bool, prefixes []string, cb func(*bootstrap.Entry) error) error { +// +// pos names dir's position; at is what is left of the resume position +// below it. +func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir, bucketRoot, pos string, at []walkFrame, versioned bool, prefixes []string, cb func(*bootstrap.Entry) error) error { skipBare := map[string]bool{} + var here walkFrame + var below []walkFrame + resuming := len(at) > 0 + if resuming { + here, below = at[0], at[1:] + } // Pass 1: .versions/ dirs only. if versioned { - if err := listAll(ctx, client, dir, func(e *filer_pb.Entry) error { + // From pass 2, only the groups whose bare file is still ahead + // matter, and they sort after here.name. + from := "" + if resuming && here.pass == passEntries { + from = here.name + } + if err := listAll(ctx, client, dir, from, false, func(e *filer_pb.Entry) error { if e == nil || e.Attributes == nil { return nil } @@ -87,17 +147,42 @@ func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir if !dirInRuleScope(prefixes, key) { return nil } - return expandVersionsDir(ctx, client, bucketRoot, key, e, start, skipBare, versioned, prefixes, cb) + var inner []walkFrame + if resuming { + if here.pass == passEntries || e.Name < here.name || (e.Name == here.name && len(below) == 0) { + // Finished earlier. Its bare null version still has + // to stay out of pass 2; for a group that had no + // versions this defers the bare file to the next walk. + skipBare[strings.TrimSuffix(key, s3_constants.VersionsFolder)] = true + return nil + } + if e.Name == here.name { + inner = below + } + } + return expandVersionsDir(ctx, client, bucketRoot, key, e, walkPosition(pos, passVersions, e.Name), inner, skipBare, versioned, prefixes, cb) }); err != nil { return err } } // Pass 2: everything else. - return listAll(ctx, client, dir, func(e *filer_pb.Entry) error { + from, inclusive := "", false + if resuming && here.pass == passEntries { + from, inclusive = here.name, true + } + return listAll(ctx, client, dir, from, inclusive, func(e *filer_pb.Entry) error { if e == nil || e.Attributes == nil { return nil } + var inner []walkFrame + if resuming && here.pass == passEntries && e.Name == here.name { + if len(below) == 0 { + return nil + } + inner = below + } + entryPos := walkPosition(pos, passEntries, e.Name) full := dir + "/" + e.Name key := strings.TrimPrefix(full, bucketRoot+"/") if e.IsDirectory && isVersionsDir(e) { @@ -109,13 +194,10 @@ func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir if !dirInRuleScope(prefixes, key) { return nil } - return walkBucketTree(ctx, client, full, bucketRoot, start, false, prefixes, cb) + return walkBucketTree(ctx, client, full, bucketRoot, entryPos, inner, false, prefixes, cb) } if e.IsDirectory { if isMPUInitDir(key, e) { - if start != "" && key <= start { - return nil - } destKey := string(e.Extended[s3_constants.ExtMultipartObjectKey]) return cb(&bootstrap.Entry{ Path: key, @@ -123,6 +205,7 @@ func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir IsMPUInit: true, ModTime: time.Unix(e.Attributes.Mtime, int64(e.Attributes.MtimeNs)), Size: int64(e.Attributes.FileSize), + Resume: entryPos, }) } // .uploads is exempt: an AbortMPU rule's prefix applies to @@ -130,14 +213,11 @@ func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir if key != s3_constants.MultipartUploadsFolder && !dirInRuleScope(prefixes, key) { return nil } - return walkBucketTree(ctx, client, full, bucketRoot, start, versioned, prefixes, cb) + return walkBucketTree(ctx, client, full, bucketRoot, entryPos, inner, versioned, prefixes, cb) } if skipBare[key] { return nil } - if start != "" && key <= start { - return nil - } entry := &bootstrap.Entry{ Path: key, ModTime: time.Unix(e.Attributes.Mtime, int64(e.Attributes.MtimeNs)), @@ -145,6 +225,7 @@ func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir IsLatest: true, // Non-versioned default. Tags: extractTags(e.Extended), Identity: router.BuildIdentityFromEntry(e), + Resume: entryPos, } return cb(entry) }) @@ -171,18 +252,16 @@ type versionItem struct { // streaming and walker paths reach the same verdict for the same // objects. Phase 5 deletes the scheduler copy. // -// Resume note: every emitted sibling shares Path = logical key, so a -// resume after a mid-expansion failure rewalks the whole sibling -// group. Acceptable today because Phase 4b doesn't persist a -// Checkpoint between runs (start is always "" via runShard). -func expandVersionsDir(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketRoot, versionsKey string, versionsEntry *filer_pb.Entry, start string, skipBare map[string]bool, versioned bool, prefixes []string, cb func(*bootstrap.Entry) error) error { +// Only the last version carries the group's position pos, so a walk +// that stops inside the group resumes before it. +func expandVersionsDir(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketRoot, versionsKey string, versionsEntry *filer_pb.Entry, pos string, at []walkFrame, skipBare map[string]bool, versioned bool, prefixes []string, cb func(*bootstrap.Entry) error) error { logical := strings.TrimSuffix(versionsKey, s3_constants.VersionsFolder) if logical == "" { return nil } versionsDir := bucketRoot + "/" + versionsKey var children []*filer_pb.Entry - if err := listAll(ctx, client, versionsDir, func(e *filer_pb.Entry) error { + if err := listAll(ctx, client, versionsDir, "", false, func(e *filer_pb.Entry) error { if e != nil && e.Attributes != nil && !e.IsDirectory { children = append(children, e) } @@ -200,7 +279,7 @@ func expandVersionsDir(ctx context.Context, client filer_pb.SeaweedFilerClient, // Coincidentally-named user folder, or an empty `.versions` // container. Treat it as a regular subdirectory so user-named // files inside still surface. - return walkBucketTree(ctx, client, versionsDir, bucketRoot, start, versioned, prefixes, cb) + return walkBucketTree(ctx, client, versionsDir, bucketRoot, pos, at, versioned, prefixes, cb) } if nullEntry, nullKey, explicit, ok := lookupNullVersion(ctx, client, bucketRoot, logical); ok { items = append(items, versionItem{ @@ -253,11 +332,6 @@ func expandVersionsDir(ctx context.Context, client filer_pb.SeaweedFilerClient, } } - if start != "" && logical <= start { - // All siblings share Path=logical, so the whole group is - // either above or below the resume marker. - return nil - } for i, it := range items { successor := s3lifecycle.SuccessorFromEntryStamp(it.entry) if successor.IsZero() && i > 0 { @@ -277,6 +351,9 @@ func expandVersionsDir(ctx context.Context, client filer_pb.SeaweedFilerClient, Tags: extractTags(it.entry.Extended), Identity: router.BuildIdentityFromEntry(it.entry), } + if i == len(items)-1 { + entry.Resume = pos + } if !isLatest { rank := i if i > latestPos { @@ -323,9 +400,8 @@ func lookupNullVersion(ctx context.Context, client filer_pb.SeaweedFilerClient, // listAll issues paginated SeaweedList calls until exhausted. Ported // from scheduler/bootstrap.go's same-named helper; Phase 5 deletes // the scheduler copy when the streaming path is removed. -func listAll(ctx context.Context, client filer_pb.SeaweedFilerClient, dir string, fn func(*filer_pb.Entry) error) error { +func listAll(ctx context.Context, client filer_pb.SeaweedFilerClient, dir, startFrom string, inclusive bool, fn func(*filer_pb.Entry) error) error { pageSize := listPageSize.Load() - startFrom := "" for { var pageCount uint32 var lastName string @@ -335,13 +411,13 @@ func listAll(ctx context.Context, client filer_pb.SeaweedFilerClient, dir string lastName = e.Name } return fn(e) - }, startFrom, false, pageSize); err != nil { + }, startFrom, inclusive, pageSize); err != nil { return err } if pageCount < pageSize { return nil } - startFrom = lastName + startFrom, inclusive = lastName, false } } diff --git a/weed/s3api/s3lifecycle/dailyrun/filer_list_func_test.go b/weed/s3api/s3lifecycle/dailyrun/filer_list_func_test.go index b5f5964f74..5d1c687632 100644 --- a/weed/s3api/s3lifecycle/dailyrun/filer_list_func_test.go +++ b/weed/s3api/s3lifecycle/dailyrun/filer_list_func_test.go @@ -86,7 +86,7 @@ func (c *fakeFiler) ListEntries(ctx context.Context, in *filer_pb.ListEntriesReq if e == nil { continue } - if in.StartFromFileName != "" && !in.InclusiveStartFrom && e.Name <= in.StartFromFileName { + if in.StartFromFileName != "" && (e.Name < in.StartFromFileName || (!in.InclusiveStartFrom && e.Name == in.StartFromFileName)) { continue } filtered = append(filtered, e) @@ -461,18 +461,105 @@ func TestFilerListFunc_VersionedDeleteMarkerPropagates(t *testing.T) { } func TestFilerListFunc_HonorsStart(t *testing.T) { - // The walker's kill-resume contract: skip entries whose Path <= start. mtime := time.Now() client := &fakeFiler{tree: map[string][]*filer_pb.Entry{ "/buckets/bkt": {file("a", mtime, 1), file("b", mtime, 1), file("c", mtime, 1)}, }} listFn := FilerListFunc(client, "/buckets", nil) var got []string - require.NoError(t, listFn(context.Background(), "bkt", "a", func(e *bootstrap.Entry) error { + require.NoError(t, listFn(context.Background(), "bkt", "2:a", func(e *bootstrap.Entry) error { got = append(got, e.Path) return nil })) assert.Equal(t, []string{"b", "c"}, got) + + got = nil + require.NoError(t, listFn(context.Background(), "bkt", "a", func(e *bootstrap.Entry) error { + got = append(got, e.Path) + return nil + })) + assert.Equal(t, []string{"a", "b", "c"}, got, "an unreadable position walks the whole bucket") +} + +func resumeTestTree() *fakeFiler { + t1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + t2 := t1.Add(time.Hour) + ver := func(id string, mtime time.Time) *filer_pb.Entry { + return fileWithExt(id, mtime, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte(id)}) + } + mpu := dir("u1") + mpu.Extended = map[string][]byte{s3_constants.ExtMultipartObjectKey: []byte("foo/big")} + vs := s3_constants.VersionsFolder + return &fakeFiler{tree: map[string][]*filer_pb.Entry{ + "/buckets/bkt": { + dir(s3_constants.MultipartUploadsFolder), + file("alpha", t1, 1), + dir("foo"), + file("foo.txt", t1, 1), + fileWithExt("zeta", t2, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte("null")}), + versionsDir("zeta"+vs, ""), + file("zz", t1, 1), + }, + "/buckets/bkt/" + s3_constants.MultipartUploadsFolder: {mpu}, + "/buckets/bkt/zeta" + vs: {ver("z1", t1)}, + "/buckets/bkt/foo": { + versionsDir("v"+vs, "v2"), + file("x", t1, 1), + }, + "/buckets/bkt/foo/v" + vs: {ver("v1", t1), ver("v2", t2)}, + }} +} + +type listedEntry struct{ Path, VersionID, Resume string } + +func listFrom(t *testing.T, listFn bootstrap.ListFunc, start string) []listedEntry { + t.Helper() + var got []listedEntry + require.NoError(t, listFn(context.Background(), "bkt", start, func(e *bootstrap.Entry) error { + got = append(got, listedEntry{e.Path, e.VersionID, e.Resume}) + return nil + })) + return got +} + +func TestFilerListFunc_ResumeFollowsTraversalOrder(t *testing.T) { + prev := listPageSize.Load() + listPageSize.Store(2) + defer listPageSize.Store(prev) + + client := resumeTestTree() + listFn := FilerListFunc(client, "/buckets", nil) + full := listFrom(t, listFn, "") + fullCalls := client.listCalls + var order []string + for _, e := range full { + order = append(order, e.Path+"@"+e.VersionID) + } + // Pass 1 groups come before pass 2 at each level, and foo's + // children before foo.txt; the null version is part of zeta's group. + require.Equal(t, []string{ + "zeta@null", "zeta@z1", + ".uploads/u1@", + "alpha@", + "foo/v@v2", "foo/v@v1", + "foo/x@", + "foo.txt@", + "zz@", + }, order) + + for i, e := range full { + if e.Resume == "" { + continue + } + assert.Equal(t, append([]listedEntry(nil), full[i+1:]...), listFrom(t, listFn, e.Resume), "resume after %s", e.Resume) + } + assert.Equal(t, "", full[0].Resume, "only a group's last version moves the position") + assert.Equal(t, "1:zeta.versions", full[1].Resume) + assert.Equal(t, "2:foo/1:v.versions", full[5].Resume) + + client.listCalls = 0 + listFrom(t, listFn, "2:foo.txt") + assert.Less(t, client.listCalls, fullCalls/2, "a resume skips listing what it already finished") } func TestFilerListFunc_NilClient(t *testing.T) { diff --git a/weed/s3api/s3lifecycle/dailyrun/run.go b/weed/s3api/s3lifecycle/dailyrun/run.go index 1adb83a144..cfb7d58b25 100644 --- a/weed/s3api/s3lifecycle/dailyrun/run.go +++ b/weed/s3api/s3lifecycle/dailyrun/run.go @@ -29,8 +29,9 @@ type LifecycleClient interface { // WalkerFunc runs one bucket walk covering every shard in shards under // the given engine view, so a whole pass costs at most a couple of -// bucket listings rather than one listing per shard. -type WalkerFunc func(ctx context.Context, view *engine.Snapshot, shards []int) error +// bucket listings rather than one listing per shard. It returns when the +// earliest bucket it covers was listed (see WalkBuckets). +type WalkerFunc func(ctx context.Context, view *engine.Snapshot, shards []int) (time.Time, error) type Config struct { Shards []int @@ -561,7 +562,7 @@ func runShard(ctx context.Context, cfg Config, snap *engine.Snapshot, runNow tim // the bug TestLifecycleExpirationDateInThePast caught. if cfg.Walker != nil && walkerDue(lastWalkedNs, runNow, cfg.WalkerInterval) { if _, walkView := snap.RulesForShard(shardID, retentionWindow); walkView != nil && len(walkView.AllActions()) > 0 { - if werr := walkForShard(ctx, cfg.Walker, walks, shardID, false, walkView); werr != nil { + if _, werr := walkForShard(ctx, cfg.Walker, walks, shardID, false, walkView); werr != nil { return fmt.Errorf("shard=%d: steady walk (empty replay): %w", shardID, werr) } lastWalkedNs = runNow.UnixNano() @@ -590,20 +591,29 @@ func runShard(ctx context.Context, cfg Config, snap *engine.Snapshot, runNow tim // via BucketBootstrapper; daily-replay needs the same. mustWalkRecovery := found && (persisted.RuleSetHash != rsh || persisted.PromotedHash != promoted) mustWalkColdStart := !found + // A resumed walk listed its earlier buckets before this pass; objects + // there that were not due yet must stay inside the replay window. + replayFrom := runNow if mustWalkRecovery || mustWalkColdStart { if cfg.Walker != nil { - if werr := walkForShard(ctx, cfg.Walker, walks, shardID, true, engine.RecoveryView(snap)); werr != nil { + since, werr := walkForShard(ctx, cfg.Walker, walks, shardID, true, engine.RecoveryView(snap)) + if werr != nil { return fmt.Errorf("shard=%d: recovery walk: %w", shardID, werr) } + if !since.IsZero() && since.Before(replayFrom) { + replayFrom = since + } lastWalkedNs = runNow.UnixNano() walkedThisPass = true } - if mustWalkRecovery { + // The shared subscription began at runNow - maxTTL, so a cold + // start whose walk resumed an earlier attempt replays next pass. + if mustWalkRecovery || replayFrom.Before(runNow) { // Rule changed: rewind cursor so the sliding replay re-scans // the new max-TTL window and the persisted hashes match the // new rule set. next := Cursor{ - TsNs: runNow.Add(-maxTTL).UnixNano(), + TsNs: replayFrom.Add(-maxTTL).UnixNano(), RuleSetHash: rsh, PromotedHash: promoted, LastWalkedNs: lastWalkedNs, @@ -627,7 +637,7 @@ func runShard(ctx context.Context, cfg Config, snap *engine.Snapshot, runNow tim // where this is a normal pass that didn't trigger cold-start. if cfg.Walker != nil && !walkedThisPass && walkerDue(lastWalkedNs, runNow, cfg.WalkerInterval) { if _, walkView := snap.RulesForShard(shardID, retentionWindow); walkView != nil && len(walkView.AllActions()) > 0 { - if werr := walkForShard(ctx, cfg.Walker, walks, shardID, false, walkView); werr != nil { + if _, werr := walkForShard(ctx, cfg.Walker, walks, shardID, false, walkView); werr != nil { return fmt.Errorf("shard=%d: steady walk: %w", shardID, werr) } lastWalkedNs = runNow.UnixNano() diff --git a/weed/s3api/s3lifecycle/dailyrun/run_bounded_test.go b/weed/s3api/s3lifecycle/dailyrun/run_bounded_test.go index ab241f1144..bd19c7e060 100644 --- a/weed/s3api/s3lifecycle/dailyrun/run_bounded_test.go +++ b/weed/s3api/s3lifecycle/dailyrun/run_bounded_test.go @@ -225,7 +225,7 @@ func TestValidate_RejectsRetentionWindowWithoutWalker(t *testing.T) { cfg := validatableConfig() cfg.RetentionWindow = 24 * time.Hour require.ErrorContains(t, validate(cfg), "RetentionWindow") - cfg.Walker = func(context.Context, *engine.Snapshot, []int) error { return nil } + cfg.Walker = func(context.Context, *engine.Snapshot, []int) (time.Time, error) { return time.Time{}, nil } require.NoError(t, validate(cfg)) cfg.Walker = nil cfg.RetentionWindow = 0 diff --git a/weed/s3api/s3lifecycle/dailyrun/run_fanout_test.go b/weed/s3api/s3lifecycle/dailyrun/run_fanout_test.go index 533689d54b..139981804b 100644 --- a/weed/s3api/s3lifecycle/dailyrun/run_fanout_test.go +++ b/weed/s3api/s3lifecycle/dailyrun/run_fanout_test.go @@ -4,6 +4,8 @@ import ( "context" "io" "strconv" + "sync" + "sync/atomic" "testing" "time" @@ -154,3 +156,90 @@ func TestRun_HaltedShardDoesNotStarveOthers(t *testing.T) { assert.Zero(t, sum.ShardErrors) assert.Equal(t, 1, sum.ShardsHalted, "a BLOCKED delete halts its shard without an error") } + +// flooredSubscribeClient serves only the events past the request's +// SinceNs, as the filer does, and records each floor it was asked for. +type flooredSubscribeClient struct { + filer_pb.SeaweedFilerClient + mu sync.Mutex + responses []*filer_pb.SubscribeMetadataResponse + floors []int64 +} + +func (c *flooredSubscribeClient) SubscribeMetadata(_ context.Context, req *filer_pb.SubscribeMetadataRequest, _ ...grpc.CallOption) (filer_pb.SeaweedFiler_SubscribeMetadataClient, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.floors = append(c.floors, req.SinceNs) + var served []*filer_pb.SubscribeMetadataResponse + for _, r := range c.responses { + if r.TsNs > req.SinceNs { + served = append(served, r) + } + } + return &scriptedSubscribeStream{responses: served}, nil +} + +func TestRun_ColdStartResumedWalkReplaysOlderEventNextPass(t *testing.T) { + const bucket = "bk" + const shard = 2 + rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1} + prior := map[s3lifecycle.ActionKey]engine.PriorState{} + ruleHash := s3lifecycle.RuleHash(rule) + for _, k := range s3lifecycle.RuleActionKinds(rule) { + prior[s3lifecycle.ActionKey{Bucket: bucket, RuleHash: ruleHash, ActionKind: k}] = engine.PriorState{ + BootstrapComplete: true, + Mode: engine.ModeEventDriven, + } + } + e := engine.New() + snap := e.Compile([]engine.CompileInput{{Bucket: bucket, Rules: []*s3lifecycle.Rule{rule}}}, + engine.CompileOptions{PriorStates: prior}) + maxTTL := engine.MaxEffectiveTTL(snap) + + runNow := time.Now().UTC() + walkedFrom := runNow.Add(-3 * maxTTL) + evTime := runNow.Add(-2 * maxTTL) + key := keyForShard(t, bucket, shard) + laterKey := "" + for i := 0; laterKey == ""; i++ { + if k := "later" + strconv.Itoa(i); s3lifecycle.ShardID(bucket, k) == shard { + laterKey = k + } + } + // The later event is inside the cold-start window; draining it in the + // first pass would move the cursor past the older one for good. + filer := &flooredSubscribeClient{responses: []*filer_pb.SubscribeMetadataResponse{ + subscribeResponse(bucket, key, evTime, evTime.UnixNano()), + subscribeResponse(bucket, laterKey, evTime, runNow.Add(-maxTTL/2).UnixNano()), + }} + client := &recordingClient{} + var walks atomic.Int32 + cfg := Config{ + Shards: []int{shard}, + BucketsPath: "/buckets", + Engine: e, + FilerClient: filer, + Client: client, + Persister: newMemPersister(), + Lister: stubSiblingLister{}, + Walker: func(context.Context, *engine.Snapshot, []int) (time.Time, error) { + if walks.Add(1) == 1 { + return walkedFrom, nil + } + return time.Time{}, nil + }, + } + + cfg.Now = func() time.Time { return runNow } + require.NoError(t, Run(context.Background(), cfg)) + assert.Empty(t, client.seenObjects(), "a cold start that resumed a walk leaves its window to the next pass") + + cfg.Now = func() time.Time { return runNow.Add(time.Minute) } + require.NoError(t, Run(context.Background(), cfg)) + require.Len(t, filer.floors, 2) + assert.Equal(t, runNow.Add(-maxTTL).UnixNano(), filer.floors[0]) + assert.Equal(t, walkedFrom.Add(-maxTTL).UnixNano(), filer.floors[1], + "the next pass subscribes from the resumed walk start") + assert.Contains(t, client.seenObjects(), key, + "the event older than the cold-start floor is replayed on the next pass") +} diff --git a/weed/s3api/s3lifecycle/dailyrun/run_stats_test.go b/weed/s3api/s3lifecycle/dailyrun/run_stats_test.go index 74e2ffca57..15ff65a9c7 100644 --- a/weed/s3api/s3lifecycle/dailyrun/run_stats_test.go +++ b/weed/s3api/s3lifecycle/dailyrun/run_stats_test.go @@ -33,8 +33,8 @@ func TestWalkBuckets_RecordsEveryBucketFailure(t *testing.T) { return nil } runStats := &RunStats{} - err := WalkBuckets(context.Background(), snapshotForBucketRule(t, "good", 7), []int{0}, - []string{"bad-a", "good", "bad-b"}, list, &recordingDispatcher{}, runStats) + _, err := WalkBuckets(context.Background(), snapshotForBucketRule(t, "good", 7), []int{0}, + []string{"bad-a", "good", "bad-b"}, list, &recordingDispatcher{}, runStats, nil) require.Error(t, err) assert.Contains(t, err.Error(), "bad-a", "the returned error stays the first bucket's") @@ -61,8 +61,8 @@ func TestWalkBuckets_CancelMarksRemainingBucketsSkipped(t *testing.T) { return nil } runStats := &RunStats{} - err := WalkBuckets(ctx, snapshotForBucketRule(t, "first", 7), []int{0}, - []string{"first", "second", "third"}, list, &recordingDispatcher{}, runStats) + _, err := WalkBuckets(ctx, snapshotForBucketRule(t, "first", 7), []int{0}, + []string{"first", "second", "third"}, list, &recordingDispatcher{}, runStats, nil) assert.ErrorIs(t, err, context.Canceled) sum := runStats.Summary() @@ -110,8 +110,9 @@ func TestWalkBuckets_CountsEntriesAndObjects(t *testing.T) { {Path: ".uploads/u1", DestKey: "c", IsDirectory: true, IsMPUInit: true, ModTime: old}, } runStats := &RunStats{} - require.NoError(t, WalkBuckets(context.Background(), snapshotForBucketRule(t, "bkt", 7), allShards(), - []string{"bkt"}, bootstrap.EntryCallback(entries), &recordingDispatcher{}, runStats)) + _, err := WalkBuckets(context.Background(), snapshotForBucketRule(t, "bkt", 7), allShards(), + []string{"bkt"}, bootstrap.EntryCallback(entries), &recordingDispatcher{}, runStats, nil) + require.NoError(t, err) sum := runStats.Summary() assert.Equal(t, int64(5), sum.EntriesVisited, "versions and upload records count, plain directories do not") @@ -169,9 +170,9 @@ func TestRunStats_NilIsANoOp(t *testing.T) { assert.Equal(t, RunSummary{}, runStats.Summary()) } -// A walk can read a bucket without visiting anything: the versioned -// pre-pass reads plain files without emitting them, and a resumed walk -// skips every key up to its start. +// A walk reads entries it does not visit: the versioned pre-pass reads +// plain files without emitting them, and a resumed walk re-reads its +// start entry. func TestRunStats_CountListedCountsSkippedEntries(t *testing.T) { mtime := time.Now().Add(-7 * 24 * time.Hour) client := &fakeFiler{tree: map[string][]*filer_pb.Entry{ @@ -179,13 +180,13 @@ func TestRunStats_CountListedCountsSkippedEntries(t *testing.T) { }} stats := &RunStats{} emitted := 0 - require.NoError(t, FilerListFunc(stats.CountListed(client), "/buckets", nil)(context.Background(), "bkt", "c.txt", func(*bootstrap.Entry) error { + require.NoError(t, FilerListFunc(stats.CountListed(client), "/buckets", nil)(context.Background(), "bkt", "2:b.txt", func(*bootstrap.Entry) error { emitted++ return nil })) - assert.Zero(t, emitted) - assert.Equal(t, int64(6), stats.Summary().EntriesListed, "both passes over the versioned bucket") + assert.Equal(t, 1, emitted) + assert.Equal(t, int64(3), stats.Summary().EntriesListed, "c.txt in the pre-pass, b.txt and c.txt in pass 2") var nilStats *RunStats assert.Same(t, client, nilStats.CountListed(client)) } diff --git a/weed/s3api/s3lifecycle/dailyrun/walk_buckets.go b/weed/s3api/s3lifecycle/dailyrun/walk_buckets.go index fe7dcfe4ad..58ab1229e0 100644 --- a/weed/s3api/s3lifecycle/dailyrun/walk_buckets.go +++ b/weed/s3api/s3lifecycle/dailyrun/walk_buckets.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" @@ -17,51 +18,66 @@ import ( // so one bucket's filer error doesn't kill the whole walk. Every // bucket's result goes to runStats when it is non-nil. // +// A non-nil progress keeps an unfinished walk's finished buckets and +// positions, and the next call for the same view and shards resumes +// from them; a completed walk clears them, so the next one is full. +// The returned time is when the earliest bucket the walk covers was +// listed: its own start, or the start of the record it resumed. +// // Composable shape: callers (the worker handler) supply a real // filer-backed ListFunc and a real Dispatcher (WalkerDispatcher). // Tests pass bootstrap.EntryCallback and a stub Dispatcher. -func WalkBuckets(ctx context.Context, view *engine.Snapshot, shards []int, buckets []string, list bootstrap.ListFunc, dispatch bootstrap.Dispatcher, runStats *RunStats) error { +func WalkBuckets(ctx context.Context, view *engine.Snapshot, shards []int, buckets []string, list bootstrap.ListFunc, dispatch bootstrap.Dispatcher, runStats *RunStats, progress WalkCheckpointPersister) (time.Time, error) { if view == nil { - return errors.New("WalkBuckets: nil view") + return time.Time{}, errors.New("WalkBuckets: nil view") } if list == nil { - return errors.New("WalkBuckets: nil list") + return time.Time{}, errors.New("WalkBuckets: nil list") } if dispatch == nil { - return errors.New("WalkBuckets: nil dispatch") + return time.Time{}, errors.New("WalkBuckets: nil dispatch") } shardSet := make(map[int]bool, len(shards)) for _, sh := range shards { shardSet[sh] = true } filtered := perShardListFunc(list, shardSet, runStats) + walk := loadWalkProgress(ctx, progress, view, shards, buckets) var firstErr error for i, b := range buckets { if err := ctx.Err(); err != nil { for _, rest := range buckets[i:] { runStats.finishBucket(rest, bucketSkipped, nil) } - return err + _ = walk.finish(ctx, err) + return walk.cp.StartedAt, err + } + if walk.done[b] { + runStats.finishBucket(b, bucketWalked, nil) + continue } runStats.startBucket(b) - _, err := bootstrap.Walk(ctx, view, b, filtered, dispatch, bootstrap.WalkOptions{}) + _, err := bootstrap.Walk(ctx, view, b, walk.track(b, filtered), dispatch, bootstrap.WalkOptions{Resume: walk.cp.Resume[b]}) switch { case err == nil: runStats.finishBucket(b, bucketWalked, nil) + walk.bucketDone(ctx, b) + continue case ctx.Err() != nil: runStats.finishBucket(b, bucketSkipped, nil) default: runStats.finishBucket(b, bucketFailed, err) } - if err != nil { - if firstErr == nil { - firstErr = fmt.Errorf("walk %s: %w", b, err) - } else { - glog.V(1).Infof("walker: additional bucket %s: %v", b, err) - } + if firstErr == nil { + firstErr = fmt.Errorf("walk %s: %w", b, err) + } else { + glog.V(1).Infof("walker: additional bucket %s: %v", b, err) } } - return firstErr + if err := walk.finish(ctx, firstErr); err != nil { + return walk.cp.StartedAt, err + } + return walk.cp.StartedAt, firstErr } // perShardListFunc wraps an inner ListFunc so only entries whose diff --git a/weed/s3api/s3lifecycle/dailyrun/walk_buckets_test.go b/weed/s3api/s3lifecycle/dailyrun/walk_buckets_test.go index c4260a395f..6e86a86b43 100644 --- a/weed/s3api/s3lifecycle/dailyrun/walk_buckets_test.go +++ b/weed/s3api/s3lifecycle/dailyrun/walk_buckets_test.go @@ -104,8 +104,8 @@ func TestWalkBuckets_DispatchesOnlyShardMatchingEntries(t *testing.T) { snap := snapshotForBucketRule(t, bucket, 7) // 30-day-old objects expire d := &recordingDispatcher{} - err := WalkBuckets(context.Background(), snap, []int{targetShard}, []string{bucket}, - bootstrap.EntryCallback(all), d, nil) + _, err := WalkBuckets(context.Background(), snap, []int{targetShard}, []string{bucket}, + bootstrap.EntryCallback(all), d, nil, nil) require.NoError(t, err) gotPaths := make([]string, 0, len(d.calls)) @@ -124,9 +124,12 @@ func TestWalkBuckets_DispatchesOnlyShardMatchingEntries(t *testing.T) { } func TestWalkBuckets_NilGuards(t *testing.T) { - require.Error(t, WalkBuckets(context.Background(), nil, []int{0}, nil, bootstrap.EntryCallback(nil), &recordingDispatcher{}, nil)) - require.Error(t, WalkBuckets(context.Background(), snapshotForBucketRule(t, "b", 7), []int{0}, nil, nil, &recordingDispatcher{}, nil)) - require.Error(t, WalkBuckets(context.Background(), snapshotForBucketRule(t, "b", 7), []int{0}, nil, bootstrap.EntryCallback(nil), nil, nil)) + _, err := WalkBuckets(context.Background(), nil, []int{0}, nil, bootstrap.EntryCallback(nil), &recordingDispatcher{}, nil, nil) + require.Error(t, err) + _, err = WalkBuckets(context.Background(), snapshotForBucketRule(t, "b", 7), []int{0}, nil, nil, &recordingDispatcher{}, nil, nil) + require.Error(t, err) + _, err = WalkBuckets(context.Background(), snapshotForBucketRule(t, "b", 7), []int{0}, nil, bootstrap.EntryCallback(nil), nil, nil, nil) + require.Error(t, err) } func TestWalkBuckets_OneBucketErrorDoesNotStopOthers(t *testing.T) { @@ -141,7 +144,7 @@ func TestWalkBuckets_OneBucketErrorDoesNotStopOthers(t *testing.T) { } snap := snapshotForBucketRule(t, "good", 7) - err := WalkBuckets(context.Background(), snap, []int{0}, []string{"bad", "good"}, list, &recordingDispatcher{}, nil) + _, err := WalkBuckets(context.Background(), snap, []int{0}, []string{"bad", "good"}, list, &recordingDispatcher{}, nil, nil) require.Error(t, err) assert.ErrorIs(t, err, listErr) } @@ -152,8 +155,8 @@ func TestWalkBuckets_HonorsContextCancellationBetweenBuckets(t *testing.T) { // short-circuit. ctx, cancel := context.WithCancel(context.Background()) cancel() - err := WalkBuckets(ctx, snapshotForBucketRule(t, "b", 7), []int{0}, []string{"b"}, - bootstrap.EntryCallback(nil), &recordingDispatcher{}, nil) + _, err := WalkBuckets(ctx, snapshotForBucketRule(t, "b", 7), []int{0}, []string{"b"}, + bootstrap.EntryCallback(nil), &recordingDispatcher{}, nil, nil) assert.ErrorIs(t, err, context.Canceled) } diff --git a/weed/s3api/s3lifecycle/dailyrun/walk_checkpoint.go b/weed/s3api/s3lifecycle/dailyrun/walk_checkpoint.go new file mode 100644 index 0000000000..440965d6f0 --- /dev/null +++ b/weed/s3api/s3lifecycle/dailyrun/walk_checkpoint.go @@ -0,0 +1,281 @@ +package dailyrun + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "sync/atomic" + "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine" +) + +// WalkCheckpoint is the saved progress of an unfinished WalkBuckets +// call. ViewHash (engine.WalkContentHash) and Shards identify the walk; +// a walk with another identity starts over. +type WalkCheckpoint struct { + ViewHash [32]byte + Shards []int + // Done lists finished buckets; Resume maps a partly walked bucket + // to its bootstrap.Checkpoint.LastScannedPath. + Done []string + Resume map[string]string + // When the walk that began this record started; every bucket in it + // was listed after that. + StartedAt time.Time + // Set by LoadWalk; SaveWalk stamps the current time. + SavedAt time.Time +} + +// WalkCheckpointPersister keeps one WalkCheckpoint per shard set. +type WalkCheckpointPersister interface { + LoadWalk(ctx context.Context, shards []int) (WalkCheckpoint, bool, error) + SaveWalk(ctx context.Context, cp WalkCheckpoint) error +} + +type walkCheckpointFile struct { + Version int `json:"version"` + Shards []int `json:"shards"` + ViewHash []byte `json:"view_hash,omitempty"` + Done []string `json:"done,omitempty"` + Resume map[string]string `json:"resume,omitempty"` + StartedNs int64 `json:"started_ns,omitempty"` + SavedNs int64 `json:"saved_ns,omitempty"` +} + +func walkCheckpointFileName(shards []int) string { + contiguous := len(shards) > 1 + for i := 1; i < len(shards); i++ { + contiguous = contiguous && shards[i] == shards[i-1]+1 + } + if contiguous { + return fmt.Sprintf("walk-%02d-%02d.json", shards[0], shards[len(shards)-1]) + } + parts := make([]string, len(shards)) + for i, sh := range shards { + parts[i] = fmt.Sprintf("%02d", sh) + } + return "walk-" + strings.Join(parts, "_") + ".json" +} + +func (p *FilerCursorPersister) LoadWalk(ctx context.Context, shards []int) (WalkCheckpoint, bool, error) { + if p.Store == nil { + return WalkCheckpoint{}, false, errors.New("FilerCursorPersister: nil Store") + } + content, err := p.Store.Read(ctx, CursorDir, walkCheckpointFileName(shards)) + if err != nil { + if errors.Is(err, filer_pb.ErrNotFound) { + return WalkCheckpoint{}, false, nil + } + return WalkCheckpoint{}, false, fmt.Errorf("walk checkpoint read: %w", err) + } + var f walkCheckpointFile + if err := json.Unmarshal(content, &f); err != nil { + return WalkCheckpoint{}, false, fmt.Errorf("walk checkpoint decode: %w", err) + } + if f.Version != cursorFileVersion { + return WalkCheckpoint{}, false, fmt.Errorf("walk checkpoint version: got %d, want %d", f.Version, cursorFileVersion) + } + if len(f.ViewHash) != 0 && len(f.ViewHash) != 32 { + return WalkCheckpoint{}, false, fmt.Errorf("walk checkpoint view_hash: got %d bytes, want 32", len(f.ViewHash)) + } + cp := WalkCheckpoint{Shards: f.Shards, Done: f.Done, Resume: f.Resume, SavedAt: time.Unix(0, f.SavedNs)} + if f.StartedNs != 0 { + cp.StartedAt = time.Unix(0, f.StartedNs) + } + copy(cp.ViewHash[:], f.ViewHash) + return cp, true, nil +} + +func (p *FilerCursorPersister) SaveWalk(ctx context.Context, cp WalkCheckpoint) error { + if p.Store == nil { + return errors.New("FilerCursorPersister: nil Store") + } + f := walkCheckpointFile{Version: cursorFileVersion, Shards: cp.Shards, Done: cp.Done, Resume: cp.Resume, SavedNs: time.Now().UnixNano()} + if !cp.StartedAt.IsZero() { + f.StartedNs = cp.StartedAt.UnixNano() + } + if cp.ViewHash != ([32]byte{}) { + f.ViewHash = cp.ViewHash[:] + } + content, err := json.Marshal(f) + if err != nil { + return fmt.Errorf("walk checkpoint encode: %w", err) + } + if err := p.Store.Save(ctx, CursorDir, walkCheckpointFileName(cp.Shards), content); err != nil { + return fmt.Errorf("walk checkpoint save: %w", err) + } + return nil +} + +// walkCheckpointInterval spaces the saves of a running walk, in ns. +var walkCheckpointInterval atomic.Int64 + +// A walk that keeps failing saves on every attempt; a record this old +// belongs to a shard set that stopped walking, and resuming it would +// skip buckets walked long ago. +const walkCheckpointMaxAge = 7 * 24 * time.Hour + +// A resumed walk that keeps failing without progress is treated as stuck +// once its record has not advanced for this long. +const walkCheckpointStallLimit = 24 * time.Hour + +func init() { walkCheckpointInterval.Store(int64(30 * time.Second)) } + +// walkProgress tracks one WalkBuckets call against its checkpoint. +type walkProgress struct { + store WalkCheckpointPersister + cp WalkCheckpoint + done map[string]bool + onDisk bool // a saved record a finished walk must clear + resumed bool + progressed bool + savedAt time.Time +} + +func loadWalkProgress(ctx context.Context, store WalkCheckpointPersister, view *engine.Snapshot, shards []int, buckets []string) *walkProgress { + sorted := slices.Clone(shards) + slices.Sort(sorted) + now := time.Now() + w := &walkProgress{ + store: store, + cp: WalkCheckpoint{ViewHash: engine.WalkContentHash(view), Shards: sorted, Resume: map[string]string{}, StartedAt: now}, + done: map[string]bool{}, + savedAt: now, + } + if store == nil { + return w + } + saved, found, err := store.LoadWalk(ctx, sorted) + if err != nil { + glog.Warningf("walker: shards %v: %v; walking every bucket", sorted, err) + w.onDisk = true + return w + } + if !found { + return w + } + w.onDisk = len(saved.Done) > 0 || len(saved.Resume) > 0 + if saved.ViewHash != w.cp.ViewHash || !slices.Equal(saved.Shards, sorted) || saved.StartedAt.IsZero() || time.Since(saved.SavedAt) > walkCheckpointMaxAge { + return w + } + listed := make(map[string]bool, len(buckets)) + for _, b := range buckets { + listed[b] = true + } + for _, b := range saved.Done { + if listed[b] { + w.done[b] = true + } + } + for b, pos := range saved.Resume { + if listed[b] && !w.done[b] { + w.cp.Resume[b] = pos + } + } + w.cp.SavedAt = saved.SavedAt + w.resumed = len(w.done) > 0 || len(w.cp.Resume) > 0 + if w.resumed { + w.cp.StartedAt = saved.StartedAt + glog.V(0).Infof("walker: shards %v resume with %d bucket(s) done and %d partly walked", sorted, len(w.done), len(w.cp.Resume)) + } + return w +} + +// track records each finished entry's position as the bucket's resume +// point, saving it every walkCheckpointInterval. +func (w *walkProgress) track(bucket string, list bootstrap.ListFunc) bootstrap.ListFunc { + if w.store == nil { + return list + } + return func(ctx context.Context, b, start string, cb func(*bootstrap.Entry) error) error { + return list(ctx, b, start, func(e *bootstrap.Entry) error { + if err := cb(e); err != nil { + return err + } + if e != nil && e.Resume != "" { + w.cp.Resume[bucket] = e.Resume + w.progressed = true + w.maybeSave(ctx) + } + return nil + }) + } +} + +func (w *walkProgress) bucketDone(ctx context.Context, bucket string) { + w.done[bucket] = true + delete(w.cp.Resume, bucket) + w.progressed = true + w.maybeSave(ctx) +} + +func (w *walkProgress) maybeSave(ctx context.Context) { + if w.store == nil || time.Since(w.savedAt) < time.Duration(walkCheckpointInterval.Load()) { + return + } + w.savedAt = time.Now() + if err := w.save(ctx); err != nil { + glog.Warningf("walker: shards %v: %v", w.cp.Shards, err) + } +} + +func (w *walkProgress) save(ctx context.Context) error { + cp := w.cp + cp.Done = make([]string, 0, len(w.done)) + for b := range w.done { + cp.Done = append(cp.Done, b) + } + slices.Sort(cp.Done) + kept := len(cp.Done) > 0 || len(cp.Resume) > 0 + // A save that reports an error may still have been written. + w.onDisk = w.onDisk || kept + if err := w.store.SaveWalk(ctx, cp); err != nil { + return err + } + w.onDisk = kept + return nil +} + +// finish saves where the walk stopped, or clears the checkpoint when it +// completed. A resumed walk that failed without progress on a record +// older than walkCheckpointStallLimit is cleared too, so one stuck +// bucket cannot keep the finished ones from ever being walked again. +// Any other failed walk without progress has nothing new to record and +// leaves the file, read or not, alone. +func (w *walkProgress) finish(ctx context.Context, walkErr error) error { + if w.store == nil { + return nil + } + reset := walkErr == nil + if walkErr != nil && !w.progressed { + if ctx.Err() != nil || !w.resumed || time.Since(w.cp.SavedAt) < walkCheckpointStallLimit { + return nil + } + glog.Warningf("walker: shards %v made no progress since %v; the next walk starts over", w.cp.Shards, w.cp.SavedAt.Format(time.RFC3339)) + reset = true + } + if reset { + clear(w.done) + clear(w.cp.Resume) + } + if !w.onDisk && len(w.done) == 0 && len(w.cp.Resume) == 0 { + return nil + } + saveCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := w.save(saveCtx); err != nil { + if walkErr == nil { + // Left in place, the record would skip buckets next walk. + return fmt.Errorf("walker: shards %v: %w", w.cp.Shards, err) + } + glog.Warningf("walker: shards %v: %v", w.cp.Shards, err) + } + return nil +} diff --git a/weed/s3api/s3lifecycle/dailyrun/walk_checkpoint_test.go b/weed/s3api/s3lifecycle/dailyrun/walk_checkpoint_test.go new file mode 100644 index 0000000000..9668bd462e --- /dev/null +++ b/weed/s3api/s3lifecycle/dailyrun/walk_checkpoint_test.go @@ -0,0 +1,355 @@ +package dailyrun + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type scriptedDispatcher struct { + calls []string + fail map[string]bool +} + +func (d *scriptedDispatcher) Delete(_ context.Context, a *engine.CompiledAction, e *bootstrap.Entry) error { + k := a.Bucket + "/" + e.Path + "@" + e.VersionID + d.calls = append(d.calls, k) + if d.fail[k] { + return errors.New("delete failed") + } + return nil +} + +type savingStore struct { + *fakeStore + saves [][]byte + attempts int + failAll bool + lostReply bool +} + +func (s *savingStore) Read(ctx context.Context, dir, name string) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return s.fakeStore.Read(ctx, dir, name) +} + +func (s *savingStore) Save(ctx context.Context, dir, name string, content []byte) error { + s.attempts++ + if s.failAll { + return errors.New("filer down") + } + s.saves = append(s.saves, content) + if err := s.fakeStore.Save(ctx, dir, name, content); err != nil { + return err + } + if s.lostReply { + return errors.New("response lost") + } + return nil +} + +func checkpointTestView(t *testing.T, prefix string) *engine.Snapshot { + t.Helper() + rules := func() []*s3lifecycle.Rule { + return []*s3lifecycle.Rule{{ID: "r", Status: s3lifecycle.StatusEnabled, Prefix: prefix, ExpirationDays: 1, NoncurrentVersionExpirationDays: 1}} + } + e := engine.New() + e.Compile([]engine.CompileInput{ + {Bucket: "a", Rules: rules()}, + {Bucket: "b", Versioned: true, Rules: rules()}, + {Bucket: "c", Rules: rules()}, + }, engine.CompileOptions{}) + snap := e.Snapshot() + for _, a := range snap.AllActions() { + snap.MarkActive(a.Key) + } + return snap +} + +func checkpointTestFiler() *fakeFiler { + t1 := time.Now().Add(-30 * 24 * time.Hour) + t2, t3 := t1.Add(time.Hour), t1.Add(2*time.Hour) + ver := func(id string, mtime time.Time) *filer_pb.Entry { + return fileWithExt(id, mtime, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte(id)}) + } + vs := s3_constants.VersionsFolder + return &fakeFiler{tree: map[string][]*filer_pb.Entry{ + "/buckets/a": {file("a1", t1, 1), file("a2", t1, 1)}, + "/buckets/b": {file("b0", t1, 1), versionsDir("k1"+vs, ""), versionsDir("obj"+vs, ""), file("z", t1, 1)}, + "/buckets/b/k1" + vs: {ver("k1a", t1), ver("k1b", t2)}, + "/buckets/b/obj" + vs: {ver("v1", t1), ver("v2", t2), ver("v3", t3)}, + "/buckets/c": {file("c1", t1, 1)}, + }} +} + +var fullWalk = []string{ + "a/a1@", "a/a2@", + "b/k1@k1b", "b/k1@k1a", "b/obj@v3", "b/obj@v2", "b/obj@v1", "b/b0@", "b/z@", + "c/c1@", +} + +type checkpointHarness struct { + t *testing.T + client *fakeFiler + store *savingStore + p *FilerCursorPersister + stats *RunStats +} + +func newCheckpointHarness(t *testing.T) *checkpointHarness { + store := &savingStore{fakeStore: newFakeStore()} + return &checkpointHarness{t: t, client: checkpointTestFiler(), store: store, p: &FilerCursorPersister{Store: store}} +} + +func (h *checkpointHarness) walk(ctx context.Context, view *engine.Snapshot, shards []int, fail ...string) ([]string, error) { + d := &scriptedDispatcher{fail: map[string]bool{}} + for _, f := range fail { + d.fail[f] = true + } + _, err := WalkBuckets(ctx, view, shards, []string{"a", "b", "c"}, FilerListFunc(h.client, "/buckets", view), d, h.stats, h.p) + return d.calls, err +} + +func (h *checkpointHarness) saved(shards []int) WalkCheckpoint { + cp, found, err := h.p.LoadWalk(context.Background(), shards) + require.NoError(h.t, err) + require.True(h.t, found) + return cp +} + +func TestWalkBuckets_ResumedWalkCountsEarlierBucketsAsWalked(t *testing.T) { + h := newCheckpointHarness(t) + view := checkpointTestView(t, "") + _, err := h.walk(context.Background(), view, allShards(), "b/obj@v1") + require.Error(t, err) + + h.stats = &RunStats{} + _, err = h.walk(context.Background(), view, allShards()) + require.NoError(t, err) + s := h.stats.Summary() + assert.Equal(t, 3, s.BucketsWalked) + assert.Zero(t, s.BucketsFailed) + assert.Equal(t, int64(5), s.EntriesVisited, "only the rest of b is visited") +} + +func TestWalkBuckets_ResumesAfterFailureInsideVersionGroup(t *testing.T) { + h := newCheckpointHarness(t) + view := checkpointTestView(t, "") + + calls, err := h.walk(context.Background(), view, allShards(), "b/obj@v1") + require.Error(t, err) + assert.Equal(t, append(fullWalk[:7:7], "c/c1@"), calls, "a failed bucket does not stop the others") + cp := h.saved(allShards()) + assert.Equal(t, []string{"a", "c"}, cp.Done) + assert.Equal(t, map[string]string{"b": "1:k1.versions"}, cp.Resume, "the position stays before the unfinished group") + assert.Equal(t, engine.WalkContentHash(view), cp.ViewHash) + + calls, err = h.walk(context.Background(), view, allShards()) + require.NoError(t, err) + assert.Equal(t, []string{"b/obj@v3", "b/obj@v2", "b/obj@v1", "b/b0@", "b/z@"}, calls) + cp = h.saved(allShards()) + assert.Empty(t, cp.Done) + assert.Empty(t, cp.Resume) + + calls, err = h.walk(context.Background(), view, allShards()) + require.NoError(t, err) + assert.Equal(t, fullWalk, calls, "after a completed walk the next one is full") +} + +func TestWalkBuckets_ResumedWalkReportsTheRecordStart(t *testing.T) { + h := newCheckpointHarness(t) + view := checkpointTestView(t, "") + walk := func(fail ...string) (time.Time, error) { + d := &scriptedDispatcher{fail: map[string]bool{}} + for _, f := range fail { + d.fail[f] = true + } + return WalkBuckets(context.Background(), view, allShards(), []string{"a", "b", "c"}, FilerListFunc(h.client, "/buckets", view), d, nil, h.p) + } + + before := time.Now() + since, err := walk("b/obj@v1") + require.Error(t, err) + assert.False(t, since.Before(before)) + assert.Equal(t, since.UnixNano(), h.saved(allShards()).StartedAt.UnixNano()) + + name := CursorDir + "/walk-00-15.json" + var f walkCheckpointFile + require.NoError(t, json.Unmarshal(h.store.files[name], &f)) + started := time.Now().Add(-48 * time.Hour) + f.StartedNs = started.UnixNano() + h.store.files[name], _ = json.Marshal(f) + since, err = walk() + require.NoError(t, err) + assert.Equal(t, started.UnixNano(), since.UnixNano(), "a resumed walk dates from the walk that listed its earlier buckets") + + before = time.Now() + since, err = walk() + require.NoError(t, err) + assert.False(t, since.Before(before), "a full walk dates from its own start") + + _, err = walk("b/obj@v1") + require.Error(t, err) + require.NoError(t, json.Unmarshal(h.store.files[name], &f)) + f.StartedNs = 0 + h.store.files[name], _ = json.Marshal(f) + before = time.Now() + since, err = walk() + require.NoError(t, err) + assert.False(t, since.Before(before), "a record without a start is not resumed") +} + +func TestWalkBuckets_CheckpointBoundToViewAndShards(t *testing.T) { + h := newCheckpointHarness(t) + view := checkpointTestView(t, "") + _, err := h.walk(context.Background(), view, allShards(), "b/obj@v1") + require.Error(t, err) + + calls, err := h.walk(context.Background(), checkpointTestView(t, "o"), allShards()) + require.NoError(t, err) + assert.Equal(t, []string{"b/obj@v3", "b/obj@v2", "b/obj@v1"}, calls, "another rule set walks from the start") + + _, err = h.walk(context.Background(), view, allShards(), "b/obj@v1") + require.Error(t, err) + var some []int + for _, sh := range allShards() { + if sh != 3 { + some = append(some, sh) + } + } + calls, err = h.walk(context.Background(), view, some) + require.NoError(t, err) + assert.Contains(t, calls, "b/k1@k1b", "another shard set walks from the start") + assert.Equal(t, map[string]string{"b": "1:k1.versions"}, h.saved(allShards()).Resume, "and keeps its own record") + assert.Contains(t, h.store.files, CursorDir+"/walk-00-15.json") + + var f walkCheckpointFile + require.NoError(t, json.Unmarshal(h.store.files[CursorDir+"/walk-00-15.json"], &f)) + f.SavedNs = time.Now().Add(-walkCheckpointMaxAge - time.Hour).UnixNano() + h.store.files[CursorDir+"/walk-00-15.json"], _ = json.Marshal(f) + calls, err = h.walk(context.Background(), view, allShards()) + require.NoError(t, err) + assert.Equal(t, fullWalk, calls, "a record nobody saved for too long is not resumed") +} + +func TestWalkBuckets_StuckResumeStartsOver(t *testing.T) { + h := newCheckpointHarness(t) + view := checkpointTestView(t, "") + _, err := h.walk(context.Background(), view, allShards(), "b/obj@v2") + require.Error(t, err) + before := h.saved(allShards()) + + calls, err := h.walk(context.Background(), view, allShards(), "b/obj@v2") + require.Error(t, err) + assert.Equal(t, []string{"b/obj@v3", "b/obj@v2"}, calls) + assert.Equal(t, before, h.saved(allShards()), "a recent record survives a resume without progress") + + var f walkCheckpointFile + require.NoError(t, json.Unmarshal(h.store.files[CursorDir+"/walk-00-15.json"], &f)) + f.SavedNs = time.Now().Add(-walkCheckpointStallLimit - time.Hour).UnixNano() + h.store.files[CursorDir+"/walk-00-15.json"], _ = json.Marshal(f) + calls, err = h.walk(context.Background(), view, allShards(), "b/obj@v2") + require.Error(t, err) + assert.Equal(t, []string{"b/obj@v3", "b/obj@v2"}, calls) + assert.Empty(t, h.saved(allShards()).Done, "a resume stuck for too long is dropped") + + calls, err = h.walk(context.Background(), view, allShards(), "b/obj@v2") + require.Error(t, err) + assert.Equal(t, fullWalk[:6], calls[:6], "so the finished buckets get walked again") +} + +func TestWalkBuckets_CanceledResumeKeepsCheckpoint(t *testing.T) { + h := newCheckpointHarness(t) + view := checkpointTestView(t, "") + _, err := h.walk(context.Background(), view, allShards(), "b/obj@v1") + require.Error(t, err) + before := h.saved(allShards()) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = h.walk(ctx, view, allShards()) + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, before, h.saved(allShards()), "a walk that could not read the record leaves it alone") +} + +func TestWalkBuckets_ClearFailureFailsTheWalk(t *testing.T) { + h := newCheckpointHarness(t) + view := checkpointTestView(t, "") + _, err := h.walk(context.Background(), view, allShards(), "b/obj@v1") + require.Error(t, err) + + h.store.failAll = true + _, err = h.walk(context.Background(), view, allShards()) + require.Error(t, err, "a record left behind would skip buckets on the next walk") + + h.store.failAll = false + calls, err := h.walk(context.Background(), view, allShards()) + require.NoError(t, err) + assert.Equal(t, []string{"b/obj@v3", "b/obj@v2", "b/obj@v1", "b/b0@", "b/z@"}, calls) +} + +func TestWalkBuckets_ResumesFromPeriodicSave(t *testing.T) { + prev := walkCheckpointInterval.Load() + walkCheckpointInterval.Store(0) + defer walkCheckpointInterval.Store(prev) + + h := newCheckpointHarness(t) + view := checkpointTestView(t, "") + calls, err := h.walk(context.Background(), view, allShards()) + require.NoError(t, err) + require.Equal(t, fullWalk, calls) + + // Replay a crash right after each periodic save. + saves := h.store.saves + require.Greater(t, len(saves), 5) + left := len(fullWalk) + for _, content := range saves[:len(saves)-1] { + var f walkCheckpointFile + require.NoError(t, json.Unmarshal(content, &f)) + h.store.files[CursorDir+"/walk-00-15.json"] = content + calls, err := h.walk(context.Background(), view, allShards()) + require.NoError(t, err) + require.LessOrEqual(t, len(calls), left, "resume after %s", content) + left = len(calls) + assert.Equal(t, append([]string(nil), fullWalk[len(fullWalk)-left:]...), calls, "resume after %s", content) + } + assert.Zero(t, left) +} + +func TestWalkProgress_FailedPeriodicSaveWaitsForTheInterval(t *testing.T) { + h := newCheckpointHarness(t) + h.store.failAll = true + w := loadWalkProgress(context.Background(), h.p, checkpointTestView(t, ""), allShards(), []string{"a"}) + w.savedAt = time.Now().Add(-time.Hour) + w.bucketDone(context.Background(), "a") + w.bucketDone(context.Background(), "a") + assert.Equal(t, 1, h.store.attempts, "a failed save is retried a full interval later, not on the next entry") +} + +func TestWalkProgress_SaveWithLostReplyIsClearedOnCompletion(t *testing.T) { + h := newCheckpointHarness(t) + h.store.lostReply = true + w := loadWalkProgress(context.Background(), h.p, checkpointTestView(t, ""), allShards(), []string{"a", "b"}) + w.savedAt = time.Now().Add(-time.Hour) + w.bucketDone(context.Background(), "a") + require.Equal(t, []string{"a"}, h.saved(allShards()).Done) + + h.store.lostReply = false + w.bucketDone(context.Background(), "b") + require.NoError(t, w.finish(context.Background(), nil)) + cp := h.saved(allShards()) + assert.Empty(t, cp.Done, "the completed walk clears the record its failed save wrote") + assert.Empty(t, cp.Resume) +} diff --git a/weed/s3api/s3lifecycle/dailyrun/walk_plan.go b/weed/s3api/s3lifecycle/dailyrun/walk_plan.go index dc51adfd9c..d95bbce7a4 100644 --- a/weed/s3api/s3lifecycle/dailyrun/walk_plan.go +++ b/weed/s3api/s3lifecycle/dailyrun/walk_plan.go @@ -27,6 +27,7 @@ func loadShardCursors(ctx context.Context, cfg Config) map[int]cursorLoad { // runShard goroutines wait on done instead of running their own listing. type pendingWalk struct { done chan struct{} + since time.Time err error recovery bool } @@ -76,7 +77,7 @@ func planWalks(ctx context.Context, cfg Config, snap *engine.Snapshot, runNow ti } go func() { defer close(pw.done) - pw.err = cfg.Walker(ctx, view, shards) + pw.since, pw.err = cfg.Walker(ctx, view, shards) }() } if len(recovery) > 0 { @@ -92,13 +93,13 @@ func planWalks(ctx context.Context, cfg Config, snap *engine.Snapshot, runNow ti // this branch (same recovery/steady kind), or runs a per-shard walk when // the plan missed — a cursor drift between the plan load and runShard's // own load — or no plan exists (direct runShard calls, tests). -func walkForShard(ctx context.Context, walker WalkerFunc, walks map[int]*pendingWalk, shardID int, recovery bool, view *engine.Snapshot) error { +func walkForShard(ctx context.Context, walker WalkerFunc, walks map[int]*pendingWalk, shardID int, recovery bool, view *engine.Snapshot) (time.Time, error) { if pw := walks[shardID]; pw != nil && pw.recovery == recovery { select { case <-pw.done: - return pw.err + return pw.since, pw.err case <-ctx.Done(): - return ctx.Err() + return time.Time{}, ctx.Err() } } return walker(ctx, view, []int{shardID}) diff --git a/weed/s3api/s3lifecycle/dailyrun/walk_plan_test.go b/weed/s3api/s3lifecycle/dailyrun/walk_plan_test.go index 7b3ffdb981..a839350f22 100644 --- a/weed/s3api/s3lifecycle/dailyrun/walk_plan_test.go +++ b/weed/s3api/s3lifecycle/dailyrun/walk_plan_test.go @@ -44,11 +44,11 @@ func TestRun_SharesBucketWalkAcrossShards(t *testing.T) { Client: stubLifecycleClient{}, Persister: p, Lister: stubSiblingLister{}, - Walker: func(_ context.Context, _ *engine.Snapshot, shards []int) error { + Walker: func(_ context.Context, _ *engine.Snapshot, shards []int) (time.Time, error) { mu.Lock() defer mu.Unlock() calls = append(calls, append([]int{}, shards...)) - return nil + return time.Time{}, nil }, } runNow := time.Unix(1_700_000_000, 0).UTC() @@ -118,12 +118,12 @@ func TestRun_RecoveryAndSteadyGroupsWalkSeparately(t *testing.T) { Client: stubLifecycleClient{}, Persister: p, Lister: stubSiblingLister{}, - Walker: func(_ context.Context, view *engine.Snapshot, shards []int) error { + Walker: func(_ context.Context, view *engine.Snapshot, shards []int) (time.Time, error) { mu.Lock() defer mu.Unlock() calls = append(calls, append([]int{}, shards...)) actions = append(actions, len(view.AllActions())) - return nil + return time.Time{}, nil }, } require.NoError(t, Run(context.Background(), cfg)) @@ -161,8 +161,8 @@ func TestRun_SharedWalkErrorFailsCoveredShards(t *testing.T) { Client: stubLifecycleClient{}, Persister: newMemPersister(), Lister: stubSiblingLister{}, - Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) error { - return errors.New("walk boom") + Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) (time.Time, error) { + return time.Time{}, errors.New("walk boom") }, } err := Run(context.Background(), cfg) @@ -212,10 +212,10 @@ func TestRun_JoinsPlannedWalkWhenShardExitsEarly(t *testing.T) { Client: stubLifecycleClient{}, Persister: &countingPersister{inner: newMemPersister(), failAfter: 1}, Lister: stubSiblingLister{}, - Walker: func(context.Context, *engine.Snapshot, []int) error { + Walker: func(context.Context, *engine.Snapshot, []int) (time.Time, error) { close(walkStarted) <-walkRelease - return nil + return time.Time{}, nil }, } runDone := make(chan error, 1) diff --git a/weed/s3api/s3lifecycle/dailyrun/walker_interval_test.go b/weed/s3api/s3lifecycle/dailyrun/walker_interval_test.go index 6cb95ebd6b..f6a9f47875 100644 --- a/weed/s3api/s3lifecycle/dailyrun/walker_interval_test.go +++ b/weed/s3api/s3lifecycle/dailyrun/walker_interval_test.go @@ -84,9 +84,9 @@ func TestRunShard_WalkerThrottle(t *testing.T) { calls := 0 cfg := Config{ Persister: p, - Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) error { + Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) (time.Time, error) { calls++ - return nil + return time.Time{}, nil }, RetentionWindow: -1, // negative sentinel: falls back to maxTTL — leaves walkView empty WalkerInterval: tc.interval, @@ -224,9 +224,9 @@ func TestRunShard_ColdStartDoesNotDoubleWalk(t *testing.T) { calls := 0 cfg := Config{ Persister: p, - Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) error { + Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) (time.Time, error) { calls++ - return nil + return time.Time{}, nil }, WalkerInterval: 0, } @@ -259,9 +259,9 @@ func TestRunShard_RecoveryWalkerSetsLastWalkedAnchor(t *testing.T) { calls := 0 cfg := Config{ Persister: p, - Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) error { + Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) (time.Time, error) { calls++ - return nil + return time.Time{}, nil }, WalkerInterval: time.Hour, } diff --git a/weed/s3api/s3lifecycle/dailyrun/walker_recovery_test.go b/weed/s3api/s3lifecycle/dailyrun/walker_recovery_test.go index 245221cdfb..9a23f7beb6 100644 --- a/weed/s3api/s3lifecycle/dailyrun/walker_recovery_test.go +++ b/weed/s3api/s3lifecycle/dailyrun/walker_recovery_test.go @@ -70,11 +70,11 @@ func TestRunShard_WalkerInvokedOnRecoveryBranch(t *testing.T) { calls := 0 cfg := Config{ Persister: p, - Walker: func(_ context.Context, view *engine.Snapshot, shards []int) error { + Walker: func(_ context.Context, view *engine.Snapshot, shards []int) (time.Time, error) { calls++ gotView = view gotShards = shards - return nil + return time.Time{}, nil }, } runNow := time.Unix(1_700_000_000, 0).UTC() @@ -93,6 +93,68 @@ func TestRunShard_WalkerInvokedOnRecoveryBranch(t *testing.T) { assert.NotEqual(t, stale, got.RuleSetHash, "stale hash must be replaced") } +func TestRunShard_RecoveryReplaysFromResumedWalkStart(t *testing.T) { + snap := snapshotWithRule(t, 30) + p := newMemPersister() + var stale [32]byte + for i := range stale { + stale[i] = 0xAD + } + require.NoError(t, p.Save(context.Background(), 3, Cursor{TsNs: 1234, RuleSetHash: stale})) + + runNow := time.Unix(1_700_000_000, 0).UTC() + walkedFrom := runNow.Add(-48 * time.Hour) + cfg := Config{ + Persister: p, + Walker: func(context.Context, *engine.Snapshot, []int) (time.Time, error) { + return walkedFrom, nil + }, + } + require.NoError(t, runShard(context.Background(), cfg, snap, runNow, 3, nil, nil)) + + got, ok, err := p.Load(context.Background(), 3) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, walkedFrom.Add(-engine.MaxEffectiveTTL(snap)).UnixNano(), got.TsNs, + "objects in buckets a resumed walk listed earlier must stay in the replay window") + assert.Equal(t, runNow.UnixNano(), got.LastWalkedNs) +} + +func TestRunShard_ColdStartWithResumedWalkReplaysNextPass(t *testing.T) { + snap := snapshotWithRule(t, 30) + p := newMemPersister() + runNow := time.Unix(1_700_000_000, 0).UTC() + walkedFrom := runNow.Add(-48 * time.Hour) + cfg := Config{ + Persister: p, + Walker: func(context.Context, *engine.Snapshot, []int) (time.Time, error) { + return walkedFrom, nil + }, + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + require.NoError(t, runShard(ctx, cfg, snap, runNow, 3, nil, nil)) + + got, ok, err := p.Load(context.Background(), 3) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, walkedFrom.Add(-engine.MaxEffectiveTTL(snap)).UnixNano(), got.TsNs, + "this pass's subscription began at runNow - maxTTL, so the next pass replays from the walk start") + assert.Equal(t, engine.ReplayContentHash(snap), got.RuleSetHash) +} + +func TestComputeGlobalStartTsNs_SavedCursorBelowTheFloor(t *testing.T) { + runNow := time.Unix(1_700_000_000, 0).UTC() + maxTTL := 30 * 24 * time.Hour + floor := runNow.Add(-maxTTL).UnixNano() + saved := runNow.Add(-maxTTL - 48*time.Hour).UnixNano() + cfg := Config{Shards: []int{1, 3}} + cursors := map[int]cursorLoad{3: {cursor: Cursor{TsNs: saved}, found: true}} + assert.Equal(t, saved, computeGlobalStartTsNs(cursors, cfg, runNow, maxTTL), + "a cursor a resumed cold start saved below runNow - maxTTL opens the next subscription") + assert.Equal(t, floor, computeGlobalStartTsNs(map[int]cursorLoad{}, cfg, runNow, maxTTL)) +} + func TestRunShard_NilWalkerOnRecoveryIsNoop(t *testing.T) { // Phase 4a behavior: a nil Walker must not crash and must still // rewind the cursor. @@ -128,8 +190,8 @@ func TestRunShard_WalkerErrorPropagates(t *testing.T) { cfg := Config{ Persister: p, - Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) error { - return errors.New("walker boom") + Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) (time.Time, error) { + return time.Time{}, errors.New("walker boom") }, } runNow := time.Unix(1_700_000_000, 0).UTC() diff --git a/weed/s3api/s3lifecycle/engine/hashes.go b/weed/s3api/s3lifecycle/engine/hashes.go index 169b62e566..8dcdbc2c04 100644 --- a/weed/s3api/s3lifecycle/engine/hashes.go +++ b/weed/s3api/s3lifecycle/engine/hashes.go @@ -171,6 +171,40 @@ func PromotedHash(s *Snapshot, retentionWindow time.Duration) [32]byte { return w.sum() } +// WalkContentHash identifies the walk a view drives: every non-disabled +// action and each of their buckets' versioning, which shapes the +// listing. Empty for a nil or empty view. +func WalkContentHash(s *Snapshot) [32]byte { + var empty [32]byte + if s == nil { + return empty + } + var items []hashItem + for k, a := range s.actions { + if a == nil || a.Mode == ModeDisabled { + continue + } + items = append(items, hashItem{key: k, action: a}) + } + if len(items) == 0 { + return empty + } + sortHashItems(items) + + w := newHashWriter() + for _, it := range items { + w.writeField(0x01, []byte(it.key.Bucket)) + w.writeField(0x02, it.key.RuleHash[:]) + w.writeInt(0x03, int64(it.key.ActionKind)) + versioned := int64(0) + if s.BucketVersioned(it.key.Bucket) { + versioned = 1 + } + w.writeInt(0x04, versioned) + } + return w.sum() +} + // MaxEffectiveTTL returns the maximum effective TTL across the *active* // replay-eligible actions in s. Returns 0 for a nil snapshot or one with no // active replay actions; the caller is expected to be in the empty-replay diff --git a/weed/s3api/s3lifecycle/engine/hashes_test.go b/weed/s3api/s3lifecycle/engine/hashes_test.go index e139ba5c30..eb7512635e 100644 --- a/weed/s3api/s3lifecycle/engine/hashes_test.go +++ b/weed/s3api/s3lifecycle/engine/hashes_test.go @@ -271,3 +271,26 @@ func TestMaxEffectiveTTL_IgnoresInactiveActions(t *testing.T) { assert.Equal(t, time.Duration(0), MaxEffectiveTTL(snap), "inactive replay actions must not contribute to MaxEffectiveTTL") } + +func TestWalkContentHash_SeparatesWalksAndEdits(t *testing.T) { + var empty [32]byte + assert.Equal(t, empty, WalkContentHash(nil)) + + rules := []*s3lifecycle.Rule{ + {ID: "days", Status: s3lifecycle.StatusEnabled, ExpirationDays: 30}, + {ID: "date", Status: s3lifecycle.StatusEnabled, Prefix: "d/", ExpirationDate: time.Now().Add(24 * time.Hour)}, + } + snap := buildSnapshotForViews(t, "b1", rules...) + _, walk := snap.RulesForShard(0, s3lifecycle.DaysToDuration(365)) + require.NotNil(t, walk) + recovery := RecoveryView(snap) + assert.NotEqual(t, WalkContentHash(walk), WalkContentHash(recovery), "steady and recovery walks cover different rules") + assert.Equal(t, WalkContentHash(recovery), WalkContentHash(RecoveryView(buildSnapshotForViews(t, "b1", rules[1], rules[0])))) + + edited := buildSnapshotForViews(t, "b1", rules[0], &s3lifecycle.Rule{ID: "date", Status: s3lifecycle.StatusEnabled, Prefix: "e/", ExpirationDate: rules[1].ExpirationDate}) + assert.NotEqual(t, WalkContentHash(recovery), WalkContentHash(RecoveryView(edited))) + + e := New() + versioned := e.Compile([]CompileInput{{Bucket: "b1", Versioned: true, Rules: rules}}, CompileOptions{}) + assert.NotEqual(t, WalkContentHash(recovery), WalkContentHash(RecoveryView(versioned)), "versioning changes the listing") +} diff --git a/weed/shell/command_s3_lifecycle_run_shard.go b/weed/shell/command_s3_lifecycle_run_shard.go index 21926fd314..2c7a653a18 100644 --- a/weed/shell/command_s3_lifecycle_run_shard.go +++ b/weed/shell/command_s3_lifecycle_run_shard.go @@ -165,9 +165,10 @@ func (c *commandS3LifecycleRunShard) Do(args []string, env *CommandEnv, writer i buckets = append(buckets, in.Bucket) } } - walker := dailyrun.WalkerFunc(func(walkCtx context.Context, view *engine.Snapshot, shards []int) error { + persister := &dailyrun.FilerCursorPersister{Store: dispatcher.NewFilerStoreClient(filerClient)} + walker := dailyrun.WalkerFunc(func(walkCtx context.Context, view *engine.Snapshot, shards []int) (time.Time, error) { return dailyrun.WalkBuckets(walkCtx, view, shards, buckets, - dailyrun.FilerListFunc(filerClient, bucketsPath, view), walkerDispatch, nil) + dailyrun.FilerListFunc(filerClient, bucketsPath, view), walkerDispatch, nil, persister) }) return dailyrun.Run(passCtx, dailyrun.Config{ Shards: shards, @@ -175,7 +176,7 @@ func (c *commandS3LifecycleRunShard) Do(args []string, env *CommandEnv, writer i Engine: eng, FilerClient: filerClient, Client: client, - Persister: &dailyrun.FilerCursorPersister{Store: dispatcher.NewFilerStoreClient(filerClient)}, + Persister: persister, Lister: dispatcher.NewFilerSiblingLister(filerClient, bucketsPath), // The shell command is used for bounded one-shot sweeps in // integration tests and CI. Fan out across the selected shards diff --git a/weed/worker/tasks/s3_lifecycle/handler.go b/weed/worker/tasks/s3_lifecycle/handler.go index ffb6aa32f3..1ce6cccd27 100644 --- a/weed/worker/tasks/s3_lifecycle/handler.go +++ b/weed/worker/tasks/s3_lifecycle/handler.go @@ -335,9 +335,9 @@ func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.Exe // Share the limiter with processMatches so walker + replay can't // combine to burst past the cluster cap. walkerDispatch := &dailyrun.WalkerDispatcher{Client: client, Limiter: limiter, Stats: runStats} - walker := dailyrun.WalkerFunc(func(walkCtx context.Context, view *engine.Snapshot, shards []int) error { + walker := dailyrun.WalkerFunc(func(walkCtx context.Context, view *engine.Snapshot, shards []int) (time.Time, error) { return dailyrun.WalkBuckets(walkCtx, view, shards, buckets, - dailyrun.FilerListFunc(runStats.CountListed(filerClient), bucketsPath, view), walkerDispatch, runStats) + dailyrun.FilerListFunc(runStats.CountListed(filerClient), bucketsPath, view), walkerDispatch, runStats, persister) }) _ = sender.SendProgress(&plugin_pb.JobProgressUpdate{