diff --git a/weed/s3api/s3api_internal_lifecycle.go b/weed/s3api/s3api_internal_lifecycle.go index 47c927d7d4..80958636e9 100644 --- a/weed/s3api/s3api_internal_lifecycle.go +++ b/weed/s3api/s3api_internal_lifecycle.go @@ -88,10 +88,19 @@ func (s3a *S3ApiServer) lifecycleDispatch(ctx context.Context, req *s3_lifecycle return resp, nil } } + // Past the attempt deadline the worker may already be resending + // this request, so a late marker write would stack a second one. + if err := ctx.Err(); err != nil { + return nil, err + } switch state { case s3_constants.VersioningEnabled: - if _, err := s3a.createDeleteMarker(req.Bucket, req.ObjectPath); err != nil { - return retryLater("TRANSPORT_ERROR: createDeleteMarker: " + err.Error()), nil + written, resp := s3a.lifecycleDeleteMarker(req.Bucket, req.ObjectPath, lifecycleAddressedVersion(req, entry)) + if resp != nil { + return resp, nil + } + if !written { + return noopResolved("LATEST_IS_DELETE_MARKER"), nil } return done(), nil case s3_constants.VersioningSuspended: @@ -122,8 +131,10 @@ func (s3a *S3ApiServer) lifecycleDispatch(ctx context.Context, req *s3_lifecycle } } } - if _, err := s3a.createDeleteMarker(req.Bucket, req.ObjectPath); err != nil { - return retryLater("TRANSPORT_ERROR: createDeleteMarker: " + err.Error()), nil + // The null delete is done, so the marker must follow even past + // the deadline: without it a retry finds an older version latest. + if _, resp := s3a.lifecycleDeleteMarker(req.Bucket, req.ObjectPath, ""); resp != nil { + return resp, nil } return done(), nil default: @@ -225,6 +236,76 @@ func (s3a *S3ApiServer) lifecycleDispatch(ctx context.Context, req *s3_lifecycle } } +// lifecycleDeleteMarker writes an expiration's delete marker as a +// compare-and-set on the .versions pointer, skipping it when the latest is +// already a delete marker. An attempt the worker gave up on may still be +// running while its retry writes, so whichever commits second sees the marker. +// A non-empty expected is the version that passed the expiration checks; a +// different latest resolves the request as stale instead of hiding it. +func (s3a *S3ApiServer) lifecycleDeleteMarker(bucket, object, expected string) (written bool, resp *s3_lifecycle_pb.LifecycleDeleteResponse) { + normalized := s3_constants.NormalizeObjectKey(object) + if s3a.objectWriteOwner(bucket, normalized) != "" { + return s3a.lifecycleDeleteMarkerIf(bucket, object, normalized, expected) + } + // Off the ring the marker commits without the guard, so the pointer read + // and the write serialize with other writers on the object write lock. + code := s3a.withObjectWriteLock(bucket, normalized, nil, func() s3err.ErrorCode { + written, resp = s3a.lifecycleDeleteMarkerIf(bucket, object, normalized, expected) + return s3err.ErrNone + }) + if code != s3err.ErrNone { + return false, retryLater("TRANSPORT_ERROR: object write lock: " + s3err.GetAPIError(code).Code) + } + return written, resp +} + +func (s3a *S3ApiServer) lifecycleDeleteMarkerIf(bucket, object, normalized, expected string) (bool, *s3_lifecycle_pb.LifecycleDeleteResponse) { + for range 3 { + pointer, err := s3a.getObjectScopedEntryRoutedByKey(bucket, normalized, s3a.bucketDir(bucket), normalized+s3_constants.VersionsFolder) + if err != nil { + if !isObjectEntryMiss(err) { + return false, retryLater("TRANSPORT_ERROR: latest-pointer lookup: " + err.Error()) + } + pointer = nil + } + if pointer != nil && string(pointer.Extended[s3_constants.ExtLatestVersionIsDeleteMarker]) == "true" && !nullVersionIsLatest(pointer) { + return false, nil + } + if expected != "" && pointer != nil { + if latest := pointerLatestVersion(pointer); latest != "" && latest != expected { + return false, noopResolved("VERSION_NOT_LATEST") + } + } + guard := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{ + Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL, + ExpectedEntry: pointer, + }}} + if _, err := s3a.createDeleteMarkerIf(bucket, object, guard); err != nil { + if errors.Is(err, errLatestVersionChanged) { + continue + } + return false, retryLater("TRANSPORT_ERROR: createDeleteMarker: " + err.Error()) + } + return true, nil + } + return false, retryLater("LATEST_CHANGED: createDeleteMarker") +} + +// pointerLatestVersion reads the latest version off a .versions pointer the +// way getLatestObjectVersion does: a named pointer with its file outranks the +// null object, the null-is-latest signal names the null version, and anything +// else leaves the latest to the caller's earlier read. +func pointerLatestVersion(pointer *filer_pb.Entry) string { + id := string(pointer.Extended[s3_constants.ExtLatestVersionIdKey]) + if id != "" && len(pointer.Extended[s3_constants.ExtLatestVersionFileNameKey]) > 0 { + return id + } + if nullVersionIsLatest(pointer) { + return "null" + } + return "" +} + // lifecycleRequireLatest refuses a current-version expiration whose version // is no longer the key's latest. A version file never changes, so the identity // check cannot see a newer write. @@ -236,22 +317,26 @@ func (s3a *S3ApiServer) lifecycleRequireLatest(req *s3_lifecycle_pb.LifecycleDel } return retryLater("TRANSPORT_ERROR: latest-version lookup: " + err.Error()) } - versionIdOrNull := func(e *filer_pb.Entry) string { - if id := string(e.Extended[s3_constants.ExtVersionIdKey]); id != "" { - return id - } - return "null" - } - addressed := req.VersionId - if addressed == "" { - addressed = versionIdOrNull(entry) - } - if versionIdOrNull(latest) != addressed { + if versionIdOrNull(latest) != lifecycleAddressedVersion(req, entry) { return noopResolved("VERSION_NOT_LATEST") } return nil } +func versionIdOrNull(e *filer_pb.Entry) string { + if id := string(e.Extended[s3_constants.ExtVersionIdKey]); id != "" { + return id + } + return "null" +} + +func lifecycleAddressedVersion(req *s3_lifecycle_pb.LifecycleDeleteRequest, entry *filer_pb.Entry) string { + if req.VersionId != "" { + return req.VersionId + } + return versionIdOrNull(entry) +} + func (s3a *S3ApiServer) lifecycleAbortMPU(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) { // req.ObjectPath is `.uploads/` (set by the router from the // init directory's bucket-relative path); reject anything that isn't diff --git a/weed/s3api/s3api_internal_lifecycle_test.go b/weed/s3api/s3api_internal_lifecycle_test.go index 147be1eba4..02b3d2768f 100644 --- a/weed/s3api/s3api_internal_lifecycle_test.go +++ b/weed/s3api/s3api_internal_lifecycle_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "net" + "strconv" "testing" "time" @@ -503,6 +504,436 @@ func TestLifecycleDelete_CurrentExpirationRequiresLatestVersion(t *testing.T) { } } +func TestLifecycleDelete_ExpiredAttemptWritesNoMarker(t *testing.T) { + latest := &filer_pb.Entry{ + Name: "v_old", + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}}, + Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")}, + } + for _, versioning := range []string{s3_constants.VersioningEnabled, s3_constants.VersioningSuspended} { + t.Run(versioning, func(t *testing.T) { + entries := map[string]*filer_pb.Entry{ + "/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{ + s3_constants.ExtLatestVersionIdKey: []byte("old"), + s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"), + }}, + "/buckets/b/obj.versions/v_old": proto.Clone(latest).(*filer_pb.Entry), + } + fake := &ambiguousPutFiler{entries: entries, apply: true} + s3a := newPutTestServer(t, startFakeFiler(t, fake)) + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: versioning}) + ctx, cancel := context.WithCancel(s3LocalPeerCtx()) + cancel() + _, err := s3a.LifecycleDelete(ctx, &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + VersionId: "old", + ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, + ExpectedIdentity: computeEntryIdentity(latest), + }) + require.ErrorIs(t, err, context.Canceled) + require.Zero(t, fake.txnCount, "an attempt past its deadline must not write a delete marker") + require.Len(t, fake.entries, 2) + }) + } +} + +func TestLifecycleDelete_DeadlineDuringNullDeleteStillWritesMarker(t *testing.T) { + olderId := generateVersionId(true) + older := &filer_pb.Entry{ + Name: "v_" + olderId, + Attributes: &filer_pb.FuseAttributes{Mtime: 1_600_000_000, FileSize: 3}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 3}}, + Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte(olderId)}, + } + latest := &filer_pb.Entry{ + Name: "obj", + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}}, + Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("null")}, + } + fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{ + "/buckets/b/obj": proto.Clone(latest).(*filer_pb.Entry), + "/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}}, + "/buckets/b/obj.versions/" + older.Name: proto.Clone(older).(*filer_pb.Entry), + }} + ctx, cancel := context.WithCancel(s3LocalPeerCtx()) + defer cancel() + fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { cancel() } + addr := startFakeFiler(t, fake) + s3a := newPutTestServer(t, addr) + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningSuspended}) + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1) + resp, err := s3a.LifecycleDelete(ctx, &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, + ExpectedIdentity: computeEntryIdentity(latest), + }) + require.NoError(t, err) + require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason) + require.Equal(t, 2, fake.txnCount, "the null delete and the marker") + got, err := s3a.getLatestObjectVersion("b", "obj") + require.NoError(t, err) + require.Equal(t, "true", string(got.Extended[s3_constants.ExtDeleteMarkerKey]), "the older version must stay hidden behind the marker") +} + +// A timed-out attempt can still be running when its retry writes the marker; +// the stale attempt's commit then finds the pointer moved and adds no second one. +func TestLifecycleDelete_OverlappingAttemptWritesOneMarker(t *testing.T) { + latest := &filer_pb.Entry{ + Name: "v_old", + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}}, + Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")}, + } + retryMarker := &filer_pb.Entry{ + Name: "v_retry", + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100}, + Extended: map[string][]byte{ + s3_constants.ExtVersionIdKey: []byte("retry"), + s3_constants.ExtDeleteMarkerKey: []byte("true"), + }, + } + cases := []struct { + versioning string + markerTxn int + outcome s3_lifecycle_pb.LifecycleDeleteOutcome + }{ + {s3_constants.VersioningEnabled, 1, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED}, + {s3_constants.VersioningSuspended, 2, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}, + } + for _, tc := range cases { + t.Run(tc.versioning, func(t *testing.T) { + fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{ + "/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{ + s3_constants.ExtLatestVersionIdKey: []byte("old"), + s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"), + }}, + "/buckets/b/obj.versions/v_old": proto.Clone(latest).(*filer_pb.Entry), + }} + fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { + if fake.txnCount != tc.markerTxn { + return + } + fake.entries["/buckets/b/obj.versions/v_retry"] = proto.Clone(retryMarker).(*filer_pb.Entry) + pointer := fake.entries["/buckets/b/obj.versions"] + pointer.Extended[s3_constants.ExtLatestVersionIdKey] = []byte("retry") + pointer.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte("v_retry") + pointer.Extended[s3_constants.ExtLatestVersionIsDeleteMarker] = []byte("true") + } + addr := startFakeFiler(t, fake) + s3a := newPutTestServer(t, addr) + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: tc.versioning}) + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1) + resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + VersionId: "old", + ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, + ExpectedIdentity: computeEntryIdentity(latest), + }) + require.NoError(t, err) + require.Equal(t, tc.outcome, resp.Outcome, resp.Reason) + var markers []string + for key, e := range fake.entries { + if string(e.Extended[s3_constants.ExtDeleteMarkerKey]) == "true" { + markers = append(markers, key) + } + } + require.Equal(t, []string{"/buckets/b/obj.versions/v_retry"}, markers) + }) + } +} + +// A version promoted after the expiration checks passed must not be hidden by +// the marker: the conflict retry finds a different latest and resolves stale. +func TestLifecycleDelete_VersionPromotedBeforeMarkerIsNotHidden(t *testing.T) { + latest := &filer_pb.Entry{ + Name: "v_old", + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}}, + Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")}, + } + promoted := &filer_pb.Entry{ + Name: "v_new", + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}}, + Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("new")}, + } + for _, versionId := range []string{"old", ""} { + t.Run("version="+versionId, func(t *testing.T) { + fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{ + "/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{ + s3_constants.ExtLatestVersionIdKey: []byte("old"), + s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"), + }}, + "/buckets/b/obj.versions/v_old": proto.Clone(latest).(*filer_pb.Entry), + }} + fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { + if fake.txnCount != 1 { + return + } + fake.entries["/buckets/b/obj.versions/v_new"] = proto.Clone(promoted).(*filer_pb.Entry) + pointer := fake.entries["/buckets/b/obj.versions"] + pointer.Extended[s3_constants.ExtLatestVersionIdKey] = []byte("new") + pointer.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte("v_new") + } + addr := startFakeFiler(t, fake) + s3a := newPutTestServer(t, addr) + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled}) + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1) + resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + VersionId: versionId, + ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, + ExpectedIdentity: computeEntryIdentity(latest), + }) + require.NoError(t, err) + require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED, resp.Outcome, resp.Reason) + require.Equal(t, "VERSION_NOT_LATEST", resp.Reason) + for key, e := range fake.entries { + require.NotEqual(t, "true", string(e.Extended[s3_constants.ExtDeleteMarkerKey]), key) + } + }) + } +} + +func TestLifecycleDelete_VersionPromotedOverNullBeforeMarkerIsNotHidden(t *testing.T) { + null := &filer_pb.Entry{ + Name: "obj", + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}}, + } + promoted := &filer_pb.Entry{ + Name: "v_new", + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}}, + Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("new")}, + } + for _, versionId := range []string{"null", ""} { + t.Run("version="+versionId, func(t *testing.T) { + fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{ + "/buckets/b/obj": proto.Clone(null).(*filer_pb.Entry), + "/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{ + s3_constants.ExtNullVersionIsLatestKey: []byte("true"), + }}, + }} + fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { + if fake.txnCount != 1 { + return + } + fake.entries["/buckets/b/obj.versions/v_new"] = proto.Clone(promoted).(*filer_pb.Entry) + pointer := fake.entries["/buckets/b/obj.versions"] + delete(pointer.Extended, s3_constants.ExtNullVersionIsLatestKey) + pointer.Extended[s3_constants.ExtLatestVersionIdKey] = []byte("new") + pointer.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte("v_new") + } + addr := startFakeFiler(t, fake) + s3a := newPutTestServer(t, addr) + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled}) + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1) + resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + VersionId: versionId, + ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, + ExpectedIdentity: computeEntryIdentity(null), + }) + require.NoError(t, err) + require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED, resp.Outcome, resp.Reason) + require.Equal(t, "VERSION_NOT_LATEST", resp.Reason) + for key, e := range fake.entries { + require.NotEqual(t, "true", string(e.Extended[s3_constants.ExtDeleteMarkerKey]), key) + } + }) + } +} + +func TestLifecycleDelete_SuspendedPutOverVersionBeforeMarkerIsNotHidden(t *testing.T) { + old := &filer_pb.Entry{ + Name: "v_old", + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}}, + Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")}, + } + fresh := &filer_pb.Entry{ + Name: "obj", + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}}, + } + for _, versionId := range []string{"old", ""} { + t.Run("version="+versionId, func(t *testing.T) { + fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{ + "/buckets/b/obj.versions/v_old": proto.Clone(old).(*filer_pb.Entry), + "/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{ + s3_constants.ExtLatestVersionIdKey: []byte("old"), + s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"), + }}, + }} + fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { + if fake.txnCount != 1 { + return + } + fake.entries["/buckets/b/obj"] = proto.Clone(fresh).(*filer_pb.Entry) + pointer := fake.entries["/buckets/b/obj.versions"] + delete(pointer.Extended, s3_constants.ExtLatestVersionIdKey) + delete(pointer.Extended, s3_constants.ExtLatestVersionFileNameKey) + pointer.Extended[s3_constants.ExtNullVersionIsLatestKey] = []byte("true") + } + addr := startFakeFiler(t, fake) + s3a := newPutTestServer(t, addr) + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled}) + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1) + resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + VersionId: versionId, + ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, + ExpectedIdentity: computeEntryIdentity(old), + }) + require.NoError(t, err) + require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED, resp.Outcome, resp.Reason) + require.Equal(t, "VERSION_NOT_LATEST", resp.Reason) + for key, e := range fake.entries { + require.NotEqual(t, "true", string(e.Extended[s3_constants.ExtDeleteMarkerKey]), key) + } + }) + } +} + +func TestLifecycleDelete_NullUnderIdOnlyPointerExpires(t *testing.T) { + null := &filer_pb.Entry{ + Name: "obj", + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}}, + } + for _, versionId := range []string{"null", ""} { + t.Run("version="+versionId, func(t *testing.T) { + fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{ + "/buckets/b/obj": proto.Clone(null).(*filer_pb.Entry), + "/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{ + s3_constants.ExtLatestVersionIdKey: []byte("old"), + }}, + }} + addr := startFakeFiler(t, fake) + s3a := newPutTestServer(t, addr) + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled}) + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1) + resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + VersionId: versionId, + ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, + ExpectedIdentity: computeEntryIdentity(null), + }) + require.NoError(t, err) + require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason) + markers := 0 + for _, e := range fake.entries { + if string(e.Extended[s3_constants.ExtDeleteMarkerKey]) == "true" { + markers++ + } + } + require.Equal(t, 1, markers) + }) + } +} + +type releaseObjectWriteLock func() + +func (r releaseObjectWriteLock) StopShortLivedLock() error { + r() + return nil +} + +// Off the ring a PUT that commits between the marker's pointer read and its +// write would be hidden; holding the object write lock makes it wait and land +// after the marker instead. +func TestLifecycleDelete_UnroutedMarkerHoldsObjectWriteLock(t *testing.T) { + old := &filer_pb.Entry{ + Name: "v_old", + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}}, + Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")}, + } + fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{ + "/buckets/b/obj.versions/v_old": proto.Clone(old).(*filer_pb.Entry), + "/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{ + s3_constants.ExtLatestVersionIdKey: []byte("old"), + s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"), + }}, + }} + interleavedId := generateVersionId(true) + put := func(id string) { + fake.entries["/buckets/b/obj.versions/v_"+id] = &filer_pb.Entry{ + Name: "v_" + id, + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}}, + Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte(id)}, + } + pointer := fake.entries["/buckets/b/obj.versions"] + pointer.Extended[s3_constants.ExtLatestVersionIdKey] = []byte(id) + pointer.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte("v_" + id) + delete(pointer.Extended, s3_constants.ExtLatestVersionIsDeleteMarker) + } + held, waiting, putDone := false, false, false + fake.beforeCreate = func() { + switch { + case putDone || waiting: + case held: + waiting = true + default: + put(interleavedId) + putDone = true + } + } + addr := startFakeFiler(t, fake) + s3a := newPutTestServer(t, addr) + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled}) + s3a.newObjectWriteLock = func(bucket, object string) (objectWriteLock, error) { + held = true + return releaseObjectWriteLock(func() { + fake.mu.Lock() + defer fake.mu.Unlock() + held = false + if waiting { + put(generateVersionId(true)) + putDone = true + } + }), nil + } + resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + VersionId: "old", + ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, + ExpectedIdentity: computeEntryIdentity(old), + }) + require.NoError(t, err) + require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason) + require.True(t, putDone) + latest, err := s3a.getLatestObjectVersion("b", "obj") + require.NoError(t, err) + require.NotEqual(t, "true", string(latest.Extended[s3_constants.ExtDeleteMarkerKey]), "the PUT must not be hidden by the marker") +} + // Expired-marker cleanup removes a marker that a lock was recorded on: markers // are never WORM-protected, so the routed delete carries no WORM guard. func TestLifecycleDelete_LockedExpiredDeleteMarkerSkipsWormGuard(t *testing.T) { @@ -552,3 +983,123 @@ func TestLifecycleDelete_LockedExpiredDeleteMarkerSkipsWormGuard(t *testing.T) { require.Len(t, conditions, 1) require.Nil(t, conditions[0], "marker delete must not carry WORM guards") } + +// A worker retries a request whose reply it lost; the replay must resolve +// against what the first attempt already did. +func TestLifecycleDelete_ReplayAfterLostReply(t *testing.T) { + version := func(id string, mtime int64, extra map[string][]byte) *filer_pb.Entry { + e := &filer_pb.Entry{ + Name: "v_" + id, + Attributes: &filer_pb.FuseAttributes{Mtime: mtime, FileSize: 3}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,0" + id + "637037d6", Size: 3}}, + Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte(id)}, + } + for k, v := range extra { + e.Extended[k] = v + } + return e + } + versionsDir := func(latestId string) *filer_pb.Entry { + return &filer_pb.Entry{ + Name: "obj" + s3_constants.VersionsFolder, + IsDirectory: true, + Attributes: &filer_pb.FuseAttributes{}, + Extended: map[string][]byte{ + s3_constants.ExtLatestVersionIdKey: []byte(latestId), + s3_constants.ExtLatestVersionFileNameKey: []byte("v_" + latestId), + }, + } + } + locked := map[string][]byte{ + s3_constants.ExtObjectLockModeKey: []byte(s3_constants.RetentionModeCompliance), + s3_constants.ExtRetentionUntilDateKey: []byte(strconv.FormatInt(time.Now().Add(24*time.Hour).Unix(), 10)), + } + oldVersion := version("1", 1_700_000_000, nil) + newVersion := version("2", 1_700_000_100, nil) + lockedVersion := version("1", 1_700_000_000, locked) + + cases := []struct { + name string + objectLock bool + kind s3_lifecycle_pb.ActionKind + target *filer_pb.Entry + entries map[string]*filer_pb.Entry + first s3_lifecycle_pb.LifecycleDeleteOutcome + replay s3_lifecycle_pb.LifecycleDeleteOutcome + }{ + {"current expiration writes one delete marker", false, s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, oldVersion, map[string]*filer_pb.Entry{ + "/buckets/b/obj.versions": versionsDir("1"), + "/buckets/b/obj.versions/v_1": oldVersion, + }, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED}, + {"noncurrent version", false, s3_lifecycle_pb.ActionKind_NONCURRENT_DAYS, oldVersion, map[string]*filer_pb.Entry{ + "/buckets/b/obj.versions": versionsDir("2"), + "/buckets/b/obj.versions/v_1": oldVersion, + "/buckets/b/obj.versions/v_2": newVersion, + }, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED}, + {"object-locked noncurrent version", true, s3_lifecycle_pb.ActionKind_NONCURRENT_DAYS, lockedVersion, map[string]*filer_pb.Entry{ + "/buckets/b/obj.versions": versionsDir("2"), + "/buckets/b/obj.versions/v_1": lockedVersion, + "/buckets/b/obj.versions/v_2": newVersion, + }, s3_lifecycle_pb.LifecycleDeleteOutcome_SKIPPED_OBJECT_LOCK, s3_lifecycle_pb.LifecycleDeleteOutcome_SKIPPED_OBJECT_LOCK}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + entries := map[string]*filer_pb.Entry{} + for k, v := range c.entries { + entries[k] = proto.Clone(v).(*filer_pb.Entry) + } + fake := &ambiguousPutFiler{entries: entries, apply: true} + addr := startFakeFiler(t, fake) + s3a := newPutTestServer(t, addr) + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + cfg := &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled} + if c.objectLock { + cfg.ObjectLockConfig = &ObjectLockConfiguration{ObjectLockEnabled: s3_constants.ObjectLockEnabled} + } + s3a.bucketConfigCache.Set("b", cfg) + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1) + + req := &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + VersionId: string(c.target.Extended[s3_constants.ExtVersionIdKey]), + ActionKind: c.kind, + ExpectedIdentity: computeEntryIdentity(c.target), + } + resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), req) + require.NoError(t, err) + require.Equal(t, c.first, resp.Outcome, resp.Reason) + + fake.mu.Lock() + afterFirst := len(fake.entries) + txns := fake.txnCount + fake.mu.Unlock() + + resp, err = s3a.LifecycleDelete(s3LocalPeerCtx(), proto.Clone(req).(*s3_lifecycle_pb.LifecycleDeleteRequest)) + require.NoError(t, err) + require.Equal(t, c.replay, resp.Outcome, resp.Reason) + + fake.mu.Lock() + defer fake.mu.Unlock() + require.Equal(t, txns, fake.txnCount, "the replay must not write") + require.Len(t, fake.entries, afterFirst) + markers := 0 + for _, e := range fake.entries { + if string(e.Extended[s3_constants.ExtDeleteMarkerKey]) == "true" { + markers++ + } + } + if c.kind == s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS { + require.Equal(t, 1, markers, "a replayed expiration must not stack delete markers") + } else { + require.Zero(t, markers) + } + if c.objectLock { + require.Contains(t, fake.entries, "/buckets/b/obj.versions/v_1", "a locked version must survive both attempts") + } else if c.kind == s3_lifecycle_pb.ActionKind_NONCURRENT_DAYS { + require.NotContains(t, fake.entries, "/buckets/b/obj.versions/v_1") + } + }) + } +} diff --git a/weed/s3api/s3api_object_handlers_copy.go b/weed/s3api/s3api_object_handlers_copy.go index 5540fa8de4..10d81a8a64 100644 --- a/weed/s3api/s3api_object_handlers_copy.go +++ b/weed/s3api/s3api_object_handlers_copy.go @@ -761,7 +761,7 @@ func (s3a *S3ApiServer) finalizeCopyDestination(dstBucket, dstObject, dstVersion // Route the pointer flip to the owner filer when known (off the // distributed lock); RECOMPUTE_LATEST picks the just-written version. if owner != "" { - if code := s3a.routedVersionedFinalize(owner, dstBucket, normalizedObject, isNewFormatVersionId(versionId), nil); code != s3err.ErrNone { + if code := s3a.routedVersionedFinalize(owner, dstBucket, normalizedObject, isNewFormatVersionId(versionId), nil, nil); code != s3err.ErrNone { if rollbackErr := s3a.rollbackCopyVersion(owner, dstBucket, normalizedObject, versionObjectPath); rollbackErr != nil { glog.Errorf("CopyObjectHandler: failed to rollback version %s for %s/%s after routed finalize error: %v", versionId, dstBucket, normalizedObject, rollbackErr) } diff --git a/weed/s3api/s3api_object_handlers_put_ambiguous_test.go b/weed/s3api/s3api_object_handlers_put_ambiguous_test.go index 8f25ef38f6..9b8a4850b5 100644 --- a/weed/s3api/s3api_object_handlers_put_ambiguous_test.go +++ b/weed/s3api/s3api_object_handlers_put_ambiguous_test.go @@ -294,6 +294,8 @@ func (f *ambiguousPutFiler) ObjectTransaction(_ context.Context, req *filer_pb.O ok = current != nil && etagCandidateMatch(clause.Etags, md5Hex) case filer_pb.WriteCondition_IF_ETAG_NOT_MATCH: ok = current == nil || !etagCandidateMatch(clause.Etags, md5Hex) + case filer_pb.WriteCondition_IF_ENTRY_EQUAL: + ok = (current == nil) == (clause.ExpectedEntry == nil) && (current == nil || proto.Equal(current, clause.ExpectedEntry)) } if !ok { return &filer_pb.ObjectTransactionResponse{ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil diff --git a/weed/s3api/s3api_object_versioned_finalize.go b/weed/s3api/s3api_object_versioned_finalize.go index a687ed4a64..99b54a2721 100644 --- a/weed/s3api/s3api_object_versioned_finalize.go +++ b/weed/s3api/s3api_object_versioned_finalize.go @@ -74,8 +74,9 @@ func (s3a *S3ApiServer) latestPointerRecompute(bucket, object string, useInverte // owner filer, via a single RECOMPUTE_LATEST. versionEntry, when set, is PUT // into .versions in the same transaction, so the recompute's scan sees it even // when the gateway's own filer is not the owner; otherwise the version file is -// already written on the owner. -func (s3a *S3ApiServer) routedVersionedFinalize(owner pb.ServerAddress, bucket, object string, useInvertedFormat bool, versionEntry *filer_pb.Entry) s3err.ErrorCode { +// already written on the owner. latestGuard, when set, must hold on the +// .versions directory entry under the same lock. +func (s3a *S3ApiServer) routedVersionedFinalize(owner pb.ServerAddress, bucket, object string, useInvertedFormat bool, versionEntry *filer_pb.Entry, latestGuard *filer_pb.WriteCondition) s3err.ErrorCode { mutations := []*filer_pb.ObjectMutation{s3a.latestPointerRecompute(bucket, object, useInvertedFormat, "", true)} if versionEntry != nil { mutations = append([]*filer_pb.ObjectMutation{{ @@ -89,11 +90,17 @@ func (s3a *S3ApiServer) routedVersionedFinalize(owner pb.ServerAddress, bucket, RouteKey: s3a.objectRouteKey(bucket, object), Mutations: mutations, } + if latestGuard != nil { + req.Condition = latestGuard + req.ConditionKey = s3a.toFilerPath(bucket, object+s3_constants.VersionsFolder) + } resp, err := s3a.objectTxnOnFiler(owner, req) switch { case err != nil: glog.Errorf("routedVersionedFinalize: %s/%s on %s: %v", bucket, object, owner, err) return s3err.ErrInternalError + case resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED: + return s3err.ErrPreconditionFailed case resp.Error != "": glog.Errorf("routedVersionedFinalize: %s/%s: %s", bucket, object, resp.Error) return s3err.ErrInternalError diff --git a/weed/s3api/s3api_object_versioning.go b/weed/s3api/s3api_object_versioning.go index 9ce68ea3d7..d2bc38a515 100644 --- a/weed/s3api/s3api_object_versioning.go +++ b/weed/s3api/s3api_object_versioning.go @@ -201,8 +201,18 @@ type ObjectVersion struct { StorageClass string } +// errLatestVersionChanged reports a delete marker refused by its latestGuard. +var errLatestVersionChanged = errors.New("latest version changed") + // createDeleteMarker creates a delete marker for versioned delete operations func (s3a *S3ApiServer) createDeleteMarker(bucket, object string) (string, error) { + return s3a.createDeleteMarkerIf(bucket, object, nil) +} + +// createDeleteMarkerIf is createDeleteMarker whose routed commit also requires +// latestGuard to hold on the .versions directory entry. The unrouted path +// has no transaction to check it in and writes unconditionally. +func (s3a *S3ApiServer) createDeleteMarkerIf(bucket, object string, latestGuard *filer_pb.WriteCondition) (string, error) { // Clean up the object path first cleanObject := strings.TrimPrefix(object, "/") @@ -239,7 +249,11 @@ func (s3a *S3ApiServer) createDeleteMarker(bucket, object string) (string, error }, Extended: deleteMarkerExtended, } - if code := s3a.routedVersionedFinalize(owner, bucket, cleanObject, useInvertedFormat, deleteMarkerEntry); code != s3err.ErrNone { + code := s3a.routedVersionedFinalize(owner, bucket, cleanObject, useInvertedFormat, deleteMarkerEntry, latestGuard) + if code == s3err.ErrPreconditionFailed { + return "", errLatestVersionChanged + } + if code != s3err.ErrNone { return "", fmt.Errorf("createDeleteMarker: routed finalize failed for %s/%s: code %d", bucket, object, code) } glog.V(2).Infof("createDeleteMarker: successfully created delete marker %s for %s/%s", versionId, bucket, object) diff --git a/weed/s3api/s3lifecycle/dailyrun/delete_retry_test.go b/weed/s3api/s3lifecycle/dailyrun/delete_retry_test.go new file mode 100644 index 0000000000..35abe6dcdd --- /dev/null +++ b/weed/s3api/s3lifecycle/dailyrun/delete_retry_test.go @@ -0,0 +1,117 @@ +package dailyrun + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/time/rate" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestDispatch_UnavailableThenSuccess(t *testing.T) { + c := &fakeLifecycleClient{scripted: []scriptedResp{ + {err: status.Error(codes.Unavailable, "connection error")}, + {outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}, + }} + out, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), 0) + require.NoError(t, err) + assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, out) + assert.Equal(t, int32(2), c.calls) +} + +func TestDispatch_ServerDeadlineIsRetried(t *testing.T) { + c := &fakeLifecycleClient{scripted: []scriptedResp{ + {err: status.Error(codes.DeadlineExceeded, "context deadline exceeded")}, + {outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}, + }} + out, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), time.Minute) + require.NoError(t, err) + assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, out) + assert.Equal(t, int32(2), c.calls) +} + +func TestDispatch_PermanentErrorsAreNotRetried(t *testing.T) { + for _, code := range []codes.Code{codes.Unauthenticated, codes.PermissionDenied, codes.InvalidArgument, codes.Unimplemented} { + t.Run(code.String(), func(t *testing.T) { + c := &fakeLifecycleClient{scripted: []scriptedResp{{err: status.Error(code, "rejected")}}} + _, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), 0) + require.Error(t, err) + assert.Equal(t, code, status.Code(err)) + assert.Equal(t, int32(1), c.calls) + }) + } +} + +func TestDispatch_NilResponseIsAnErrorNotAPanic(t *testing.T) { + c := &walkerStubClient{nilResp: true} + _, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), 0) + require.ErrorIs(t, err, errNilDeleteResponse) +} + +func TestDispatch_EveryAttemptWaitsOnLimiter(t *testing.T) { + lim := rate.NewLimiter(rate.Every(time.Hour), transportRetryAttempts) + c := &fakeLifecycleClient{scripted: []scriptedResp{ + {err: status.Error(codes.Unavailable, "down")}, + {err: status.Error(codes.Unavailable, "down")}, + {outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}, + }} + _, err := dispatchWithRetry(context.Background(), c, lim, sampleMatch(), 0) + require.NoError(t, err) + assert.Equal(t, int32(transportRetryAttempts), c.calls) + assert.Less(t, lim.Tokens(), 0.5, "each attempt must take a token from the shared budget") +} + +func TestProcessMatches_RetryExhaustionHoldsCursorBeforeTheMatch(t *testing.T) { + runNow := time.Now() + first, second := sampleMatch(), sampleMatch() + first.DueTime, second.DueTime = runNow.Add(-time.Hour), runNow.Add(-time.Hour) + second.ObjectKey = "obj2" + c := &fakeLifecycleClient{scripted: []scriptedResp{ + {outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}, + {err: status.Error(codes.Unavailable, "down")}, + }} + _, halted, err := processMatches(context.Background(), Config{Client: c}, runNow, &reader.Event{TsNs: runNow.UnixNano()}, []router.Match{first, second}) + require.Error(t, err) + assert.True(t, halted) + assert.Equal(t, int32(1+transportRetryAttempts), c.calls) +} + +func TestWalkerDispatcher_TransientErrorIsRetried(t *testing.T) { + c := &fakeLifecycleClient{scripted: []scriptedResp{ + {err: status.Error(codes.Unavailable, "connection reset")}, + {outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}, + }} + d := &WalkerDispatcher{Client: c} + err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"}) + require.NoError(t, err) + assert.Equal(t, int32(2), c.calls) +} + +func TestWalkerDispatcher_AuthErrorSurfacesAtOnce(t *testing.T) { + c := &fakeLifecycleClient{scripted: []scriptedResp{{err: status.Error(codes.Unauthenticated, "invalid admin token")}}} + d := &WalkerDispatcher{Client: c} + err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"}) + require.Error(t, err) + assert.Equal(t, codes.Unauthenticated, status.Code(err)) + assert.Equal(t, int32(1), c.calls) +} + +func TestWalkerDispatcher_ParentCancelDuringBackoffStops(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + c := &fakeLifecycleClient{scripted: []scriptedResp{{err: errors.New("transport: connection refused")}}} + time.AfterFunc(10*time.Millisecond, cancel) + d := &WalkerDispatcher{Client: c} + err := d.Delete(ctx, sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"}) + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, int32(1), c.calls) +} diff --git a/weed/s3api/s3lifecycle/dailyrun/delete_timeout_test.go b/weed/s3api/s3lifecycle/dailyrun/delete_timeout_test.go new file mode 100644 index 0000000000..21171ba597 --- /dev/null +++ b/weed/s3api/s3lifecycle/dailyrun/delete_timeout_test.go @@ -0,0 +1,155 @@ +package dailyrun + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/time/rate" + "google.golang.org/protobuf/proto" +) + +// stallingClient blocks each attempt until its context ends, except +// attempts listed in answerAfter, which answer DONE after that delay. +type stallingClient struct { + mu sync.Mutex + requests []*s3_lifecycle_pb.LifecycleDeleteRequest + answerAfter map[int]time.Duration + onCall func(attempt int) +} + +func (c *stallingClient) LifecycleDelete(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) { + c.mu.Lock() + c.requests = append(c.requests, req) + attempt := len(c.requests) + delay, answers := c.answerAfter[attempt] + c.mu.Unlock() + if c.onCall != nil { + c.onCall(attempt) + } + if answers { + select { + case <-time.After(delay): + return &s3_lifecycle_pb.LifecycleDeleteResponse{Outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}, nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + <-ctx.Done() + return nil, ctx.Err() +} + +func (c *stallingClient) calls() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.requests) +} + +func versionedMatch() router.Match { + m := sampleMatch() + m.Key.ActionKind = s3lifecycle.ActionKindNoncurrentDays + m.VersionID = "v_1" + m.Identity = &router.EntryIdentity{MtimeNs: 1, Size: 7, HeadFid: "3,01", ExtendedHash: []byte{0xaa}} + return m +} + +func TestDispatch_StalledAttemptsTimeOutAndFail(t *testing.T) { + c := &stallingClient{} + start := time.Now() + _, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), 20*time.Millisecond) + require.Error(t, err) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Contains(t, err.Error(), "timed out after 20ms") + assert.Equal(t, transportRetryAttempts, c.calls(), "an attempt timeout is retried like a transport error") + assert.Less(t, time.Since(start), 5*time.Second) +} + +func TestDispatch_SlowAttemptWithinTimeoutSucceeds(t *testing.T) { + c := &stallingClient{answerAfter: map[int]time.Duration{1: 30 * time.Millisecond}} + out, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), time.Second) + require.NoError(t, err) + assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, out) + assert.Equal(t, 1, c.calls()) +} + +func TestDispatch_RetryAfterTimeoutResendsSameRequest(t *testing.T) { + c := &stallingClient{answerAfter: map[int]time.Duration{2: 0}} + out, err := dispatchWithRetry(context.Background(), c, nil, versionedMatch(), 20*time.Millisecond) + require.NoError(t, err) + assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, out) + require.Len(t, c.requests, 2) + assert.True(t, proto.Equal(c.requests[0], c.requests[1]), "a retry after a lost reply must keep the version and identity witness") + assert.Equal(t, "v_1", c.requests[1].VersionId) + assert.Equal(t, "3,01", c.requests[1].ExpectedIdentity.GetHeadFid()) +} + +func TestDispatch_ParentCancelInterruptsActiveAttempt(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + c := &stallingClient{onCall: func(int) { time.AfterFunc(20*time.Millisecond, cancel) }} + start := time.Now() + _, err := dispatchWithRetry(ctx, c, nil, sampleMatch(), time.Hour) + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, c.calls(), "parent cancellation must not be retried") + assert.Less(t, time.Since(start), 5*time.Second) +} + +func TestDispatch_ParentCancelInterruptsBackoff(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + c := &fakeLifecycleClient{scripted: []scriptedResp{{err: errors.New("transport boom")}}} + time.AfterFunc(10*time.Millisecond, cancel) + _, err := dispatchWithRetry(ctx, c, nil, sampleMatch(), time.Hour) + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, int32(1), c.calls, "cancellation during backoff must stop before the next attempt") +} + +func TestProcessMatches_AttemptTimeoutFailsShard(t *testing.T) { + runNow := time.Now() + m := sampleMatch() + m.DueTime = runNow.Add(-time.Hour) + c := &stallingClient{} + cfg := Config{Client: c, DeleteTimeout: 10 * time.Millisecond} + _, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{TsNs: runNow.UnixNano()}, []router.Match{m}) + require.ErrorIs(t, err, context.DeadlineExceeded, "exhausted attempts must fail the pass, not report it clean") + assert.True(t, halted, "an unresolved delete must hold the cursor") +} + +func TestProcessMatches_LimiterWaitDoesNotSpendDeleteTimeout(t *testing.T) { + runNow := time.Now() + m := sampleMatch() + m.DueTime = runNow.Add(-time.Hour) + lim := rate.NewLimiter(rate.Every(150*time.Millisecond), 1) + _ = lim.AllowN(time.Now(), 1) + c := &stallingClient{answerAfter: map[int]time.Duration{1: 20 * time.Millisecond}} + cfg := Config{Client: c, Limiter: lim, DeleteTimeout: 100 * time.Millisecond} + _, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{TsNs: runNow.UnixNano()}, []router.Match{m}) + require.NoError(t, err) + assert.False(t, halted) + assert.Equal(t, 1, c.calls()) +} + +func TestWalkerDispatcher_AttemptTimeoutHaltsWalk(t *testing.T) { + c := &stallingClient{} + d := &WalkerDispatcher{Client: c, DeleteTimeout: 10 * time.Millisecond} + err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "timed out after 10ms") + assert.Equal(t, transportRetryAttempts, c.calls(), "the walker shares the replay retry budget") +} + +func TestWalkerDispatcher_LimiterWaitDoesNotSpendDeleteTimeout(t *testing.T) { + lim := rate.NewLimiter(rate.Every(150*time.Millisecond), 1) + _ = lim.AllowN(time.Now(), 1) + c := &stallingClient{answerAfter: map[int]time.Duration{1: 20 * time.Millisecond}} + d := &WalkerDispatcher{Client: c, Limiter: lim, DeleteTimeout: 100 * time.Millisecond} + err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"}) + require.NoError(t, err) +} diff --git a/weed/s3api/s3lifecycle/dailyrun/dispatch.go b/weed/s3api/s3lifecycle/dailyrun/dispatch.go index 542a1d73bc..baa06ce37a 100644 --- a/weed/s3api/s3lifecycle/dailyrun/dispatch.go +++ b/weed/s3api/s3lifecycle/dailyrun/dispatch.go @@ -3,19 +3,22 @@ package dailyrun import ( "context" "errors" + "fmt" "math/rand" "time" "github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router" + "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/util" + "golang.org/x/time/rate" + "google.golang.org/grpc/codes" ) -// transportRetryAttempts is the in-run retry budget for transport -// errors (RPC dial failures, stream resets). Server-side outcomes -// (RETRY_LATER, BLOCKED) are NOT retried in-run — the server already -// classified them as "wait until next run." The retry exists only to -// paper over a single flake from the network. +// transportRetryAttempts is the in-run retry budget shared by the walker +// and replay deletes. Only transport failures and attempt timeouts are +// retried; server outcomes (RETRY_LATER, BLOCKED) wait for the next run. const transportRetryAttempts = 3 // transportRetryInitial is the first sleep after a failed attempt. @@ -26,41 +29,89 @@ const ( transportRetryMax = 5 * time.Second ) -// dispatchWithRetry sends one LifecycleDelete request, retrying on -// transport errors up to transportRetryAttempts times with exponential -// backoff. Returns the server's outcome on success, or an error if all -// retries exhausted. Server outcomes (RETRY_LATER / BLOCKED) bypass -// the retry — the daily run's halt-on-failure semantics handles them. -func dispatchWithRetry(ctx context.Context, client LifecycleClient, m router.Match) (s3_lifecycle_pb.LifecycleDeleteOutcome, error) { - req := buildDeleteRequest(m) +// DefaultDeleteTimeout bounds one LifecycleDelete attempt when the +// caller sets no timeout. +const DefaultDeleteTimeout = time.Minute + +var ( + errDeleteAttemptTimedOut = errors.New("lifecycle delete attempt timed out") + errNilDeleteResponse = errors.New("nil response") +) + +// deleteOnce sends one LifecycleDelete attempt under its own deadline, +// so a stalled RPC fails that attempt instead of holding the shard for +// the job's lifetime. +func deleteOnce(ctx context.Context, client LifecycleClient, req *s3_lifecycle_pb.LifecycleDeleteRequest, timeout time.Duration) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) { + if timeout <= 0 { + timeout = DefaultDeleteTimeout + } + attemptCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + resp, err := client.LifecycleDelete(attemptCtx, req) + if err != nil && ctx.Err() == nil && attemptCtx.Err() != nil { + err = fmt.Errorf("%w after %s: %w", errDeleteAttemptTimedOut, timeout, err) + } + if err == nil && resp == nil { + err = errNilDeleteResponse + } + return resp, err +} + +// isRetryableDeleteError reports whether a failed attempt may be resent. +// Authentication, permission and argument errors fail the same way +// again, so they surface at once. +func isRetryableDeleteError(err error) bool { + if errors.Is(err, errDeleteAttemptTimedOut) || util.IsTransientError(err) { + return true + } + st, ok := util.ServerStatus(err) + return ok && st.Code() == codes.DeadlineExceeded +} + +// deleteWithRetry sends req, retrying transport failures and attempt +// timeouts up to transportRetryAttempts times with jittered exponential +// backoff. Every attempt waits on limiter, so retries stay inside the +// shared delete budget. The resent request keeps its version and identity +// witness, so a delete that committed before its reply was lost resolves +// as NOOP_RESOLVED on the retry. +func deleteWithRetry(ctx context.Context, client LifecycleClient, limiter *rate.Limiter, req *s3_lifecycle_pb.LifecycleDeleteRequest, timeout time.Duration) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) { backoff := transportRetryInitial - var lastErr error - for attempt := 1; attempt <= transportRetryAttempts; attempt++ { - resp, err := client.LifecycleDelete(ctx, req) + for attempt := 1; ; attempt++ { + if limiter != nil { + waitStart := time.Now() + if err := limiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("limiter: %w", err) + } + stats.S3LifecycleDispatchLimiterWaitSeconds.Observe(time.Since(waitStart).Seconds()) + } + resp, err := deleteOnce(ctx, client, req, timeout) if err == nil { - return resp.Outcome, nil + return resp, nil } - // Context cancellation is shutdown, not a transport flake. - // Surface it immediately so the caller halts and tomorrow - // resumes from the same cursor. - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, err + // Parent cancellation is shutdown, not a transport flake. + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr } - lastErr = err - if attempt == transportRetryAttempts { - break + if attempt >= transportRetryAttempts || !isRetryableDeleteError(err) { + return nil, err } select { case <-ctx.Done(): - return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, ctx.Err() + return nil, ctx.Err() case <-time.After(jitter(backoff)): } - backoff *= 2 - if backoff > transportRetryMax { - backoff = transportRetryMax - } + backoff = min(backoff*2, transportRetryMax) } - return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, lastErr +} + +// dispatchWithRetry sends the LifecycleDelete for m under the shared +// retry policy and returns the server's outcome. +func dispatchWithRetry(ctx context.Context, client LifecycleClient, limiter *rate.Limiter, m router.Match, timeout time.Duration) (s3_lifecycle_pb.LifecycleDeleteOutcome, error) { + resp, err := deleteWithRetry(ctx, client, limiter, buildDeleteRequest(m), timeout) + if err != nil { + return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, err + } + return resp.Outcome, nil } // jitter returns a duration in the range [d/2, d) using equal jitter. diff --git a/weed/s3api/s3lifecycle/dailyrun/dispatch_test.go b/weed/s3api/s3lifecycle/dailyrun/dispatch_test.go index 33131b16a4..b18a546944 100644 --- a/weed/s3api/s3lifecycle/dailyrun/dispatch_test.go +++ b/weed/s3api/s3lifecycle/dailyrun/dispatch_test.go @@ -56,7 +56,7 @@ func dispatchWithRetryFast(t *testing.T, ctx context.Context, c LifecycleClient, // We can't override the package-private constants, but every test // path here uses small attempt counts so even the production // backoff is fine. - return dispatchWithRetry(ctx, c, m) + return dispatchWithRetry(ctx, c, nil, m, 0) } func TestDispatch_FirstAttemptSucceeds(t *testing.T) { diff --git a/weed/s3api/s3lifecycle/dailyrun/run.go b/weed/s3api/s3lifecycle/dailyrun/run.go index cfb7d58b25..9106582082 100644 --- a/weed/s3api/s3lifecycle/dailyrun/run.go +++ b/weed/s3api/s3lifecycle/dailyrun/run.go @@ -48,6 +48,10 @@ type Config struct { // nil -> no rate limit. Shared across all shard goroutines. Limiter *rate.Limiter + // DeleteTimeout bounds each LifecycleDelete attempt, applied after + // the limiter wait. <= 0 uses DefaultDeleteTimeout. + DeleteTimeout time.Duration + // Meta-log retention boundary. Rules whose effective TTL exceeds // this can't be serviced by replay alone and get partitioned into // the walk view (engine.PromotedHash). 0 falls back to maxTTL, @@ -675,8 +679,9 @@ func runShard(ctx context.Context, cfg Config, snap *engine.Snapshot, runNow tim if drainErr != nil { _ = saveCursorAndPublish(saveCtx, cfg.Persister, shardID, Cursor{TsNs: lastOK, RuleSetHash: rsh, PromotedHash: promoted, LastWalkedNs: lastWalkedNs}) // Cut short by its wall-clock cap: truncated, not failed — the - // cursor above carries progress forward. - if errors.Is(drainErr, context.DeadlineExceeded) || errors.Is(drainErr, context.Canceled) { + // cursor above carries progress forward. A timed-out delete + // attempt also wraps DeadlineExceeded, so judge by ctx itself. + if ctx.Err() != nil { return nil } return fmt.Errorf("shard=%d: drain: %w", shardID, drainErr) @@ -807,14 +812,7 @@ func processMatches(ctx context.Context, cfg Config, runNow time.Time, ev *reade skippedAny = true continue } - if cfg.Limiter != nil { - waitStart := time.Now() - if waitErr := cfg.Limiter.Wait(ctx); waitErr != nil { - return skippedAny, true, waitErr - } - stats.S3LifecycleDispatchLimiterWaitSeconds.Observe(time.Since(waitStart).Seconds()) - } - outcome, dispatchErr := dispatchWithRetry(ctx, cfg.Client, m) + outcome, dispatchErr := dispatchWithRetry(ctx, cfg.Client, cfg.Limiter, m, cfg.DeleteTimeout) if dispatchErr != nil { glog.V(1).Infof("daily_run: transport error on %s/%s %s: %v", m.Bucket, m.ObjectKey, m.Key.ActionKind, dispatchErr) @@ -822,7 +820,7 @@ func processMatches(ctx context.Context, cfg Config, runNow time.Time, ev *reade // (dispatcher/dispatcher.go) so transport failures // aggregate under one outcome key across paths. countDispatch(cfg.Stats, m.Bucket, m.Key.ActionKind.String(), "RPC_ERROR") - return skippedAny, true, nil + return skippedAny, true, fmt.Errorf("%s/%s %s: %w", m.Bucket, m.ObjectKey, m.Key.ActionKind, dispatchErr) } countDispatch(cfg.Stats, m.Bucket, m.Key.ActionKind.String(), outcome.String()) switch outcome { diff --git a/weed/s3api/s3lifecycle/dailyrun/walker_dispatcher.go b/weed/s3api/s3lifecycle/dailyrun/walker_dispatcher.go index 6ac582ba23..8092b0b259 100644 --- a/weed/s3api/s3lifecycle/dailyrun/walker_dispatcher.go +++ b/weed/s3api/s3lifecycle/dailyrun/walker_dispatcher.go @@ -8,7 +8,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine" - "github.com/seaweedfs/seaweedfs/weed/stats" "golang.org/x/time/rate" ) @@ -24,6 +23,9 @@ type WalkerDispatcher struct { // daily-run's processMatches uses so the walker and replay paths // can't combine to burst past the cap. nil disables throttling. Limiter *rate.Limiter + // DeleteTimeout bounds each LifecycleDelete, applied after the + // limiter wait. <= 0 uses DefaultDeleteTimeout. + DeleteTimeout time.Duration // nil skips per-pass outcome counts. Stats *RunStats } @@ -66,27 +68,13 @@ func (d *WalkerDispatcher) Delete(ctx context.Context, action *engine.CompiledAc ExpectedIdentity: toProtoIdentity(entry.Identity), } kindLabel := action.Key.ActionKind.String() - if d.Limiter != nil { - waitStart := time.Now() - if waitErr := d.Limiter.Wait(ctx); waitErr != nil { - return fmt.Errorf("walker dispatch %s/%s %s: limiter: %w", action.Bucket, objectPath, action.Key.ActionKind, waitErr) - } - stats.S3LifecycleDispatchLimiterWaitSeconds.Observe(time.Since(waitStart).Seconds()) - } - resp, err := d.Client.LifecycleDelete(ctx, req) + resp, err := deleteWithRetry(ctx, d.Client, d.Limiter, req, d.DeleteTimeout) if err != nil { // "RPC_ERROR" matches the streaming dispatcher and processMatches // labels so transport-class failures aggregate under one key. countDispatch(d.Stats, action.Bucket, kindLabel, "RPC_ERROR") return fmt.Errorf("walker dispatch %s/%s %s: %w", action.Bucket, objectPath, action.Key.ActionKind, err) } - if resp == nil { - // A misbehaving server stub returning (nil, nil) would panic on - // the switch below. Bucket under RPC_ERROR rather than a new - // label — operationally it's the same class of failure. - countDispatch(d.Stats, action.Bucket, kindLabel, "RPC_ERROR") - return fmt.Errorf("walker dispatch %s/%s %s: nil response", action.Bucket, objectPath, action.Key.ActionKind) - } countDispatch(d.Stats, action.Bucket, kindLabel, resp.Outcome.String()) switch resp.Outcome { case s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, diff --git a/weed/worker/tasks/s3_lifecycle/cluster_rate_limit.go b/weed/worker/tasks/s3_lifecycle/cluster_rate_limit.go index 34866a38e7..857f826b22 100644 --- a/weed/worker/tasks/s3_lifecycle/cluster_rate_limit.go +++ b/weed/worker/tasks/s3_lifecycle/cluster_rate_limit.go @@ -42,6 +42,9 @@ const ( // the per-shard walk budget — typically 60 (1h) for small clusters, // 360+ (6h+) for large ones. WalkerIntervalMinutesAdminKey = "walker_interval_minutes" + // DeleteTimeoutSecondsAdminKey bounds each LifecycleDelete attempt. + // 0 = dailyrun.DefaultDeleteTimeout. + DeleteTimeoutSecondsAdminKey = "delete_timeout_seconds" // MetadataKeyDeletesPerSecond is the per-worker share value the // admin writes into ClusterContext.Metadata at ExecuteJob time. diff --git a/weed/worker/tasks/s3_lifecycle/config.go b/weed/worker/tasks/s3_lifecycle/config.go index 6b21c2bcb3..b0c156eb1a 100644 --- a/weed/worker/tasks/s3_lifecycle/config.go +++ b/weed/worker/tasks/s3_lifecycle/config.go @@ -21,6 +21,9 @@ type Config struct { // behavior; positive values gate the walker via Cursor.LastWalkedNs // inside dailyrun.runShard. WalkerInterval time.Duration + // DeleteTimeout bounds each LifecycleDelete attempt; 0 uses + // dailyrun.DefaultDeleteTimeout. + DeleteTimeout time.Duration } func ParseConfig(adminValues map[string]*plugin_pb.ConfigValue, workerValues map[string]*plugin_pb.ConfigValue) Config { @@ -42,6 +45,9 @@ func ParseConfig(adminValues map[string]*plugin_pb.ConfigValue, workerValues map if mins := readInt64(adminValues, WalkerIntervalMinutesAdminKey, 0); mins > 0 { cfg.WalkerInterval = time.Duration(mins) * time.Minute } + if secs := readInt64(adminValues, DeleteTimeoutSecondsAdminKey, 0); secs > 0 { + cfg.DeleteTimeout = time.Duration(secs) * time.Second + } return cfg } diff --git a/weed/worker/tasks/s3_lifecycle/config_test.go b/weed/worker/tasks/s3_lifecycle/config_test.go index 4b774f345e..e8a9d79838 100644 --- a/weed/worker/tasks/s3_lifecycle/config_test.go +++ b/weed/worker/tasks/s3_lifecycle/config_test.go @@ -93,3 +93,21 @@ func TestParseConfigWalkerIntervalNegativeStaysZero(t *testing.T) { t.Errorf("negative WalkerInterval should stay 0, got %v", cfg.WalkerInterval) } } + +func TestParseConfig_DeleteTimeout(t *testing.T) { + if cfg := ParseConfig(nil, nil); cfg.DeleteTimeout != 0 { + t.Errorf("DeleteTimeout default=%v, want 0 (dailyrun default)", cfg.DeleteTimeout) + } + cfg := ParseConfig(map[string]*plugin_pb.ConfigValue{ + DeleteTimeoutSecondsAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 45}}, + }, nil) + if want := 45 * time.Second; cfg.DeleteTimeout != want { + t.Errorf("DeleteTimeout=%v, want %v", cfg.DeleteTimeout, want) + } + cfg = ParseConfig(map[string]*plugin_pb.ConfigValue{ + DeleteTimeoutSecondsAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: -5}}, + }, nil) + if cfg.DeleteTimeout != 0 { + t.Errorf("negative DeleteTimeout should stay 0, got %v", cfg.DeleteTimeout) + } +} diff --git a/weed/worker/tasks/s3_lifecycle/handler.go b/weed/worker/tasks/s3_lifecycle/handler.go index f597f3679e..8a80fef9a0 100644 --- a/weed/worker/tasks/s3_lifecycle/handler.go +++ b/weed/worker/tasks/s3_lifecycle/handler.go @@ -111,6 +111,14 @@ func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor { Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER, MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, }, + { + Name: DeleteTimeoutSecondsAdminKey, + Label: "Delete RPC Timeout (seconds)", + Description: "Deadline for each lifecycle delete RPC to an S3 server, counted after the rate-limit wait. Replay and walker deletes get up to 3 attempts. After that a replay shard stops at that event and resumes there on the next run, and a walk fails and runs again from the start on the next run. 0 = 60 seconds.", + FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64, + Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER, + MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, + }, }, }, }, @@ -119,6 +127,7 @@ func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor { ClusterDeletesBurstAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, MetaLogRetentionDaysAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, WalkerIntervalMinutesAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, + DeleteTimeoutSecondsAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, }, }, // WorkerConfigForm intentionally absent — the prior @@ -336,7 +345,7 @@ func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.Exe runStats := &dailyrun.RunStats{} // Share the limiter with processMatches so walker + replay can't // combine to burst past the cluster cap. - walkerDispatch := &dailyrun.WalkerDispatcher{Client: client, Limiter: limiter, Stats: runStats} + walkerDispatch := &dailyrun.WalkerDispatcher{Client: client, Limiter: limiter, DeleteTimeout: cfg.DeleteTimeout, Stats: runStats} walker := dailyrun.WalkerFunc(func(walkCtx context.Context, view *engine.Snapshot, shards []int) (time.Time, error) { return dailyrun.WalkBuckets(walkCtx, view, shards, buckets, dailyrun.FilerListFunc(runStats.CountListed(filerClient), bucketsPath, view), walkerDispatch, runStats, persister) @@ -360,6 +369,7 @@ func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.Exe Lister: dispatcher.NewFilerSiblingLister(filerClient, bucketsPath), Workers: cfg.Workers, Limiter: limiter, + DeleteTimeout: cfg.DeleteTimeout, RetentionWindow: cfg.MetaLogRetention, Walker: walker, WalkerInterval: cfg.WalkerInterval,