feat(s3/lifecycle): WalkerDispatcher adapter for the daily-run walker

Phase 4b prep. Implements bootstrap.Dispatcher on top of LifecycleClient
so the same LifecycleDelete RPC drives both the meta-log replay path
and the walker. No CAS witness — the server's identityMatches treats
nil ExpectedIdentity as a bootstrap call and rebuilds the witness from
the live entry, which is the right contract for a full-tree walk.

Adds VersionID to bootstrap.Entry so versioned-bucket walks address
the right version. MPU init uses DestKey for ObjectPath (matching the
prefix-match contract); rejecting empty DestKey keeps malformed init
records out of the dispatch path.

Not wired yet — runShard still doesn't invoke the walker. Follow-up
commits add the ListFunc adapter and the recovery-branch wiring.
This commit is contained in:
Chris Lu committed 2026-05-11 21:51:45 -07:00
1 parent 2c4bd59281
commit b624b6edd1
3 files changed
+226

No files matched your search

@@ -40,6 +40,13 @@ type Entry struct {
SuccessorModTime time.Time
NoncurrentIndex *int
// VersionID is the S3 version id of this entry, empty for
// non-versioned buckets. Populated by the ListFunc adapter when
// the entry came from a `.versions/` directory; the walker itself
// doesn't use it, but the Dispatcher needs it to address the
// right version on LifecycleDelete.
VersionID string
}
// ListFunc must skip entries with Path <= start so kill-resume picks up
@@ -0,0 +1,74 @@
package dailyrun
import (
"context"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
)
// WalkerDispatcher adapts LifecycleClient to bootstrap.Dispatcher so
// the Phase 4b walker can drive the same LifecycleDelete RPC the
// meta-log replay path uses. No CAS witness is supplied; the server's
// identityMatches treats nil ExpectedIdentity as "bootstrap call, skip
// witness" (see weed/s3api/s3api_internal_lifecycle.go), which is the
// right contract for a full-tree walk that has just observed the entry.
type WalkerDispatcher struct {
Client LifecycleClient
}
// Compile-time check.
var _ bootstrap.Dispatcher = (*WalkerDispatcher)(nil)
// Delete classifies non-DONE server outcomes as errors so the walker
// halts and the caller persists progress under the bootstrap
// checkpoint rather than silently skipping objects.
func (d *WalkerDispatcher) Delete(ctx context.Context, action *engine.CompiledAction, entry *bootstrap.Entry) error {
if d == nil || d.Client == nil {
return fmt.Errorf("walker dispatch: nil client")
}
if action == nil || entry == nil {
return fmt.Errorf("walker dispatch: nil action or entry")
}
objectPath := entry.Path
if entry.IsMPUInit {
// Rule-prefix matching used DestKey; the server takes the
// canonical object path for the LifecycleDelete RPC, which
// is also DestKey. The walker hits the .uploads/<id>
// directory itself only when ActionKind=ABORT_MPU, and the
// server resolves the upload from (bucket, object_path) +
// the init record's metadata.
if entry.DestKey == "" {
return fmt.Errorf("walker dispatch: MPU init entry with empty DestKey: %s", entry.Path)
}
objectPath = entry.DestKey
}
rh := action.Key.RuleHash
req := &s3_lifecycle_pb.LifecycleDeleteRequest{
Bucket: action.Bucket,
ObjectPath: objectPath,
VersionId: entry.VersionID,
RuleHash: rh[:],
ActionKind: toProtoActionKind(action.Key.ActionKind),
// ExpectedIdentity intentionally nil; server bootstraps from
// the live entry on this code path.
}
resp, err := d.Client.LifecycleDelete(ctx, req)
if err != nil {
return fmt.Errorf("walker dispatch %s/%s %s: %w", action.Bucket, objectPath, action.Key.ActionKind, err)
}
switch resp.Outcome {
case s3_lifecycle_pb.LifecycleDeleteOutcome_DONE,
s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED,
s3_lifecycle_pb.LifecycleDeleteOutcome_SKIPPED_OBJECT_LOCK:
return nil
default:
// RETRY_LATER / BLOCKED / UNSPECIFIED: surface as error so the
// walk halts at this entry and resumes from
// Checkpoint.LastScannedPath on the next run.
return fmt.Errorf("walker dispatch %s/%s %s: outcome=%s reason=%s",
action.Bucket, objectPath, action.Key.ActionKind, resp.Outcome, resp.Reason)
}
}
@@ -0,0 +1,145 @@
package dailyrun
import (
"context"
"errors"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// walkerStubClient captures the last LifecycleDeleteRequest so tests
// can assert on the request shape produced by WalkerDispatcher.
type walkerStubClient struct {
lastReq *s3_lifecycle_pb.LifecycleDeleteRequest
outcome s3_lifecycle_pb.LifecycleDeleteOutcome
err error
reason string
}
func (c *walkerStubClient) LifecycleDelete(_ context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
c.lastReq = req
if c.err != nil {
return nil, c.err
}
return &s3_lifecycle_pb.LifecycleDeleteResponse{Outcome: c.outcome, Reason: c.reason}, nil
}
func sampleAction(t *testing.T, kind s3lifecycle.ActionKind) *engine.CompiledAction {
t.Helper()
var rh [8]byte
for i := range rh {
rh[i] = byte(0xa0 + i)
}
return &engine.CompiledAction{
Bucket: "bkt",
Key: s3lifecycle.ActionKey{Bucket: "bkt", ActionKind: kind, RuleHash: rh},
Mode: engine.ModeEventDriven,
}
}
func TestWalkerDispatcher_NonVersionedSendsExpectedRequest(t *testing.T) {
c := &walkerStubClient{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}
d := &WalkerDispatcher{Client: c}
a := sampleAction(t, s3lifecycle.ActionKindExpirationDays)
err := d.Delete(context.Background(), a, &bootstrap.Entry{Path: "obj"})
require.NoError(t, err)
require.NotNil(t, c.lastReq)
assert.Equal(t, "bkt", c.lastReq.Bucket)
assert.Equal(t, "obj", c.lastReq.ObjectPath)
assert.Equal(t, "", c.lastReq.VersionId)
assert.Equal(t, a.Key.RuleHash[:], c.lastReq.RuleHash)
assert.Equal(t, s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, c.lastReq.ActionKind)
// Bootstrap-style call: server skips CAS witness when nil.
assert.Nil(t, c.lastReq.ExpectedIdentity)
}
func TestWalkerDispatcher_VersionedPassesVersionID(t *testing.T) {
c := &walkerStubClient{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}
d := &WalkerDispatcher{Client: c}
a := sampleAction(t, s3lifecycle.ActionKindNoncurrentDays)
err := d.Delete(context.Background(), a, &bootstrap.Entry{Path: "obj", VersionID: "v-abc"})
require.NoError(t, err)
assert.Equal(t, "v-abc", c.lastReq.VersionId)
}
func TestWalkerDispatcher_MPUInitUsesDestKey(t *testing.T) {
// Rule-prefix matching used DestKey; the RPC ObjectPath must match
// so the server resolves the upload from the user's intended key.
c := &walkerStubClient{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}
d := &WalkerDispatcher{Client: c}
a := sampleAction(t, s3lifecycle.ActionKindAbortMPU)
err := d.Delete(context.Background(), a, &bootstrap.Entry{
Path: ".uploads/abc123",
DestKey: "user/path/object",
IsMPUInit: true,
})
require.NoError(t, err)
assert.Equal(t, "user/path/object", c.lastReq.ObjectPath)
}
func TestWalkerDispatcher_MPUInitEmptyDestKeyErrors(t *testing.T) {
// An MPU init record with no DestKey is mid-write before metadata
// landed; skipping silently in the walker is fine, but the
// dispatcher must not invent a path.
c := &walkerStubClient{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}
d := &WalkerDispatcher{Client: c}
a := sampleAction(t, s3lifecycle.ActionKindAbortMPU)
err := d.Delete(context.Background(), a, &bootstrap.Entry{
Path: ".uploads/abc123",
IsMPUInit: true,
})
require.Error(t, err)
assert.Nil(t, c.lastReq, "no RPC should be sent when DestKey is empty")
}
func TestWalkerDispatcher_AcceptsAllResolvedOutcomes(t *testing.T) {
for _, oc := range []s3_lifecycle_pb.LifecycleDeleteOutcome{
s3_lifecycle_pb.LifecycleDeleteOutcome_DONE,
s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED,
s3_lifecycle_pb.LifecycleDeleteOutcome_SKIPPED_OBJECT_LOCK,
} {
c := &walkerStubClient{outcome: oc}
d := &WalkerDispatcher{Client: c}
err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
assert.NoError(t, err, "outcome %s must be treated as resolved", oc)
}
}
func TestWalkerDispatcher_UnresolvedOutcomeReturnsError(t *testing.T) {
// RETRY_LATER, BLOCKED, and UNSPECIFIED all halt the walk so it
// resumes from Checkpoint.LastScannedPath on the next run.
for _, oc := range []s3_lifecycle_pb.LifecycleDeleteOutcome{
s3_lifecycle_pb.LifecycleDeleteOutcome_RETRY_LATER,
s3_lifecycle_pb.LifecycleDeleteOutcome_BLOCKED,
s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED,
} {
c := &walkerStubClient{outcome: oc, reason: "server said so"}
d := &WalkerDispatcher{Client: c}
err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
require.Error(t, err, "outcome %s must halt the walk", oc)
assert.Contains(t, err.Error(), oc.String())
}
}
func TestWalkerDispatcher_TransportErrorReturnsWrappedError(t *testing.T) {
c := &walkerStubClient{err: errors.New("transport boom")}
d := &WalkerDispatcher{Client: c}
err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
require.Error(t, err)
assert.Contains(t, err.Error(), "transport boom")
}
func TestWalkerDispatcher_NilGuardsReturnError(t *testing.T) {
d := &WalkerDispatcher{Client: &walkerStubClient{}}
require.Error(t, d.Delete(context.Background(), nil, &bootstrap.Entry{Path: "obj"}))
require.Error(t, d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), nil))
nilClient := &WalkerDispatcher{}
require.Error(t, nilClient.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"}))
}