mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 00:38:53 +01:00
s3 lifecycle: bound each LifecycleDelete attempt with its own deadline (#12646)
* s3 lifecycle: bound each LifecycleDelete attempt with its own deadline Replay and walker deletes used the job context with no request deadline, so one stalled unary RPC could hold a shard for the whole job. Each attempt now runs under DeleteTimeout (default 1m), set after the limiter wait. A timeout while the job is still live is retried like a transport error and then halts the shard or walk without advancing; parent cancellation still stops both the attempt and the backoff. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: admin setting for the delete RPC timeout Adds delete_timeout_seconds to the S3 Lifecycle admin form and passes it to both the replay drain and the walker. 0 keeps the 60s default. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: describe replay and walker timeouts separately in the help text Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: fail the pass when a replay delete exhausts its attempts A transport error or attempt timeout that outlived the retries halted the shard with a nil error, so the job reported success. processMatches now returns the error (the cursor still holds at the event), and runShard treats a drain error as a clean stop only when its own context ended, since an attempt timeout also wraps DeadlineExceeded. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: test that a replayed LifecycleDelete resolves after a lost reply Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: skip the expiration write once the attempt deadline passed A worker retries a LifecycleDelete whose attempt timed out, while the first attempt can still be running on the S3 server. An attempt that stalled before its marker write would then stack a second delete marker after the retry wrote one. The gRPC deadline reaches the handler context, so check it right before the expiration writes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: recheck the attempt deadline after the suspended null delete The null-version delete can block on the owner filer past the attempt deadline; the worker may have retried and written a marker by then. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: finish the suspended marker once the null version is deleted Skipping the marker after a successful null delete left an older version as the latest, and the retry then resolved as a no-op without hiding it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3: let a routed delete marker commit only while its latest guard holds createDeleteMarkerIf passes an optional condition, evaluated on the .versions directory entry under the object's lock on the owner filer, and reports a refused commit as errLatestVersionChanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: write the expiration marker as a compare-and-set on the latest pointer A timed-out attempt can still be running on the S3 server while the worker's retry writes the marker. The marker commit is now conditioned on the .versions pointer as read just before it and skipped when the latest is already a delete marker, so whichever attempt commits second writes nothing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: resolve the marker as stale when another version became latest The marker's compare-and-set only covered changes after its own pointer read, so a version promoted after the expiration checks passed could be hidden by the marker. The enabled path now passes the validated version and a conflict retry that finds a different named latest resolves the request as VERSION_NOT_LATEST. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: resolve a null expiration as stale once a version takes the pointer Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: read the marker's latest version the way getLatestObjectVersion does A pointer names a version only with its file name; an id alone falls back to the null object, and the null-is-latest signal makes the null version current. An expiration that validated a named version no longer hides a null written by a suspended PUT, and a current null under an id-only pointer now expires instead of resolving as stale. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: hold the object write lock for an unrouted expiration marker Off the ring createDeleteMarkerIf cannot carry the pointer guard, so a PUT committing between the pointer read and the marker write was hidden. The read and the write now run under the object write lock that unrouted writers take. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: share transient-error retries between walker and replay deletes (#12649) * s3 lifecycle: retry only transient LifecycleDelete failures dispatchWithRetry retried every RPC error except parent cancellation, so an authentication, permission or argument error spent the whole retry budget before surfacing. Retry only attempt timeouts and the failures util.IsTransientError recognizes (Unavailable, ResourceExhausted, network resets); everything else returns on the first attempt. A nil response is now an error instead of a nil dereference. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: take a delete-rate token for every LifecycleDelete attempt processMatches waited on the shared limiter once per match, so the retries behind it ran outside the cluster-wide deletes/sec budget. The retry loop now waits on the limiter before each attempt. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: give walker deletes the replay retry policy WalkerDispatcher sent each LifecycleDelete once, so one dropped connection aborted the bucket's daily walk while the same failure on the replay path was retried. The walker now goes through deleteWithRetry and shares its transient-error retries, per-attempt limiter wait and nil response handling. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: retry a DeadlineExceeded the server sends back A server or proxy can end the RPC with DeadlineExceeded while the attempt's own deadline is still running. util.IsTransientError rejects that status, so the delete halted with retries left. It is a timed-out attempt like the client-side one, so retry it the same way. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: say walker deletes get 3 attempts in the timeout help Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
504de9b562
commit
63806b7073
16 files changed
+1083
-78
No files matched your search
@@ -88,10 +88,19 @@ func (s3a *S3ApiServer) lifecycleDispatch(ctx context.Context, req *s3_lifecycle
|
||||
return resp, nil
|
||||
}
|
||||
}
|
||||
// Past the attempt deadline the worker may already be resending
|
||||
// this request, so a late marker write would stack a second one.
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch state {
|
||||
case s3_constants.VersioningEnabled:
|
||||
if _, err := s3a.createDeleteMarker(req.Bucket, req.ObjectPath); err != nil {
|
||||
return retryLater("TRANSPORT_ERROR: createDeleteMarker: " + err.Error()), nil
|
||||
written, resp := s3a.lifecycleDeleteMarker(req.Bucket, req.ObjectPath, lifecycleAddressedVersion(req, entry))
|
||||
if resp != nil {
|
||||
return resp, nil
|
||||
}
|
||||
if !written {
|
||||
return noopResolved("LATEST_IS_DELETE_MARKER"), nil
|
||||
}
|
||||
return done(), nil
|
||||
case s3_constants.VersioningSuspended:
|
||||
@@ -122,8 +131,10 @@ func (s3a *S3ApiServer) lifecycleDispatch(ctx context.Context, req *s3_lifecycle
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := s3a.createDeleteMarker(req.Bucket, req.ObjectPath); err != nil {
|
||||
return retryLater("TRANSPORT_ERROR: createDeleteMarker: " + err.Error()), nil
|
||||
// The null delete is done, so the marker must follow even past
|
||||
// the deadline: without it a retry finds an older version latest.
|
||||
if _, resp := s3a.lifecycleDeleteMarker(req.Bucket, req.ObjectPath, ""); resp != nil {
|
||||
return resp, nil
|
||||
}
|
||||
return done(), nil
|
||||
default:
|
||||
@@ -225,6 +236,76 @@ func (s3a *S3ApiServer) lifecycleDispatch(ctx context.Context, req *s3_lifecycle
|
||||
}
|
||||
}
|
||||
|
||||
// lifecycleDeleteMarker writes an expiration's delete marker as a
|
||||
// compare-and-set on the .versions pointer, skipping it when the latest is
|
||||
// already a delete marker. An attempt the worker gave up on may still be
|
||||
// running while its retry writes, so whichever commits second sees the marker.
|
||||
// A non-empty expected is the version that passed the expiration checks; a
|
||||
// different latest resolves the request as stale instead of hiding it.
|
||||
func (s3a *S3ApiServer) lifecycleDeleteMarker(bucket, object, expected string) (written bool, resp *s3_lifecycle_pb.LifecycleDeleteResponse) {
|
||||
normalized := s3_constants.NormalizeObjectKey(object)
|
||||
if s3a.objectWriteOwner(bucket, normalized) != "" {
|
||||
return s3a.lifecycleDeleteMarkerIf(bucket, object, normalized, expected)
|
||||
}
|
||||
// Off the ring the marker commits without the guard, so the pointer read
|
||||
// and the write serialize with other writers on the object write lock.
|
||||
code := s3a.withObjectWriteLock(bucket, normalized, nil, func() s3err.ErrorCode {
|
||||
written, resp = s3a.lifecycleDeleteMarkerIf(bucket, object, normalized, expected)
|
||||
return s3err.ErrNone
|
||||
})
|
||||
if code != s3err.ErrNone {
|
||||
return false, retryLater("TRANSPORT_ERROR: object write lock: " + s3err.GetAPIError(code).Code)
|
||||
}
|
||||
return written, resp
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) lifecycleDeleteMarkerIf(bucket, object, normalized, expected string) (bool, *s3_lifecycle_pb.LifecycleDeleteResponse) {
|
||||
for range 3 {
|
||||
pointer, err := s3a.getObjectScopedEntryRoutedByKey(bucket, normalized, s3a.bucketDir(bucket), normalized+s3_constants.VersionsFolder)
|
||||
if err != nil {
|
||||
if !isObjectEntryMiss(err) {
|
||||
return false, retryLater("TRANSPORT_ERROR: latest-pointer lookup: " + err.Error())
|
||||
}
|
||||
pointer = nil
|
||||
}
|
||||
if pointer != nil && string(pointer.Extended[s3_constants.ExtLatestVersionIsDeleteMarker]) == "true" && !nullVersionIsLatest(pointer) {
|
||||
return false, nil
|
||||
}
|
||||
if expected != "" && pointer != nil {
|
||||
if latest := pointerLatestVersion(pointer); latest != "" && latest != expected {
|
||||
return false, noopResolved("VERSION_NOT_LATEST")
|
||||
}
|
||||
}
|
||||
guard := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{
|
||||
Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL,
|
||||
ExpectedEntry: pointer,
|
||||
}}}
|
||||
if _, err := s3a.createDeleteMarkerIf(bucket, object, guard); err != nil {
|
||||
if errors.Is(err, errLatestVersionChanged) {
|
||||
continue
|
||||
}
|
||||
return false, retryLater("TRANSPORT_ERROR: createDeleteMarker: " + err.Error())
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return false, retryLater("LATEST_CHANGED: createDeleteMarker")
|
||||
}
|
||||
|
||||
// pointerLatestVersion reads the latest version off a .versions pointer the
|
||||
// way getLatestObjectVersion does: a named pointer with its file outranks the
|
||||
// null object, the null-is-latest signal names the null version, and anything
|
||||
// else leaves the latest to the caller's earlier read.
|
||||
func pointerLatestVersion(pointer *filer_pb.Entry) string {
|
||||
id := string(pointer.Extended[s3_constants.ExtLatestVersionIdKey])
|
||||
if id != "" && len(pointer.Extended[s3_constants.ExtLatestVersionFileNameKey]) > 0 {
|
||||
return id
|
||||
}
|
||||
if nullVersionIsLatest(pointer) {
|
||||
return "null"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// lifecycleRequireLatest refuses a current-version expiration whose version
|
||||
// is no longer the key's latest. A version file never changes, so the identity
|
||||
// check cannot see a newer write.
|
||||
@@ -236,22 +317,26 @@ func (s3a *S3ApiServer) lifecycleRequireLatest(req *s3_lifecycle_pb.LifecycleDel
|
||||
}
|
||||
return retryLater("TRANSPORT_ERROR: latest-version lookup: " + err.Error())
|
||||
}
|
||||
versionIdOrNull := func(e *filer_pb.Entry) string {
|
||||
if id := string(e.Extended[s3_constants.ExtVersionIdKey]); id != "" {
|
||||
return id
|
||||
}
|
||||
return "null"
|
||||
}
|
||||
addressed := req.VersionId
|
||||
if addressed == "" {
|
||||
addressed = versionIdOrNull(entry)
|
||||
}
|
||||
if versionIdOrNull(latest) != addressed {
|
||||
if versionIdOrNull(latest) != lifecycleAddressedVersion(req, entry) {
|
||||
return noopResolved("VERSION_NOT_LATEST")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func versionIdOrNull(e *filer_pb.Entry) string {
|
||||
if id := string(e.Extended[s3_constants.ExtVersionIdKey]); id != "" {
|
||||
return id
|
||||
}
|
||||
return "null"
|
||||
}
|
||||
|
||||
func lifecycleAddressedVersion(req *s3_lifecycle_pb.LifecycleDeleteRequest, entry *filer_pb.Entry) string {
|
||||
if req.VersionId != "" {
|
||||
return req.VersionId
|
||||
}
|
||||
return versionIdOrNull(entry)
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) lifecycleAbortMPU(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
|
||||
// req.ObjectPath is `.uploads/<upload_id>` (set by the router from the
|
||||
// init directory's bucket-relative path); reject anything that isn't
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -503,6 +504,436 @@ func TestLifecycleDelete_CurrentExpirationRequiresLatestVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_ExpiredAttemptWritesNoMarker(t *testing.T) {
|
||||
latest := &filer_pb.Entry{
|
||||
Name: "v_old",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")},
|
||||
}
|
||||
for _, versioning := range []string{s3_constants.VersioningEnabled, s3_constants.VersioningSuspended} {
|
||||
t.Run(versioning, func(t *testing.T) {
|
||||
entries := map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte("old"),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"),
|
||||
}},
|
||||
"/buckets/b/obj.versions/v_old": proto.Clone(latest).(*filer_pb.Entry),
|
||||
}
|
||||
fake := &ambiguousPutFiler{entries: entries, apply: true}
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, fake))
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: versioning})
|
||||
ctx, cancel := context.WithCancel(s3LocalPeerCtx())
|
||||
cancel()
|
||||
_, err := s3a.LifecycleDelete(ctx, &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: "old",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(latest),
|
||||
})
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
require.Zero(t, fake.txnCount, "an attempt past its deadline must not write a delete marker")
|
||||
require.Len(t, fake.entries, 2)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_DeadlineDuringNullDeleteStillWritesMarker(t *testing.T) {
|
||||
olderId := generateVersionId(true)
|
||||
older := &filer_pb.Entry{
|
||||
Name: "v_" + olderId,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_600_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte(olderId)},
|
||||
}
|
||||
latest := &filer_pb.Entry{
|
||||
Name: "obj",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("null")},
|
||||
}
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj": proto.Clone(latest).(*filer_pb.Entry),
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
|
||||
"/buckets/b/obj.versions/" + older.Name: proto.Clone(older).(*filer_pb.Entry),
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(s3LocalPeerCtx())
|
||||
defer cancel()
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { cancel() }
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningSuspended})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
resp, err := s3a.LifecycleDelete(ctx, &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(latest),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason)
|
||||
require.Equal(t, 2, fake.txnCount, "the null delete and the marker")
|
||||
got, err := s3a.getLatestObjectVersion("b", "obj")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "true", string(got.Extended[s3_constants.ExtDeleteMarkerKey]), "the older version must stay hidden behind the marker")
|
||||
}
|
||||
|
||||
// A timed-out attempt can still be running when its retry writes the marker;
|
||||
// the stale attempt's commit then finds the pointer moved and adds no second one.
|
||||
func TestLifecycleDelete_OverlappingAttemptWritesOneMarker(t *testing.T) {
|
||||
latest := &filer_pb.Entry{
|
||||
Name: "v_old",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")},
|
||||
}
|
||||
retryMarker := &filer_pb.Entry{
|
||||
Name: "v_retry",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100},
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtVersionIdKey: []byte("retry"),
|
||||
s3_constants.ExtDeleteMarkerKey: []byte("true"),
|
||||
},
|
||||
}
|
||||
cases := []struct {
|
||||
versioning string
|
||||
markerTxn int
|
||||
outcome s3_lifecycle_pb.LifecycleDeleteOutcome
|
||||
}{
|
||||
{s3_constants.VersioningEnabled, 1, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED},
|
||||
{s3_constants.VersioningSuspended, 2, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.versioning, func(t *testing.T) {
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte("old"),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"),
|
||||
}},
|
||||
"/buckets/b/obj.versions/v_old": proto.Clone(latest).(*filer_pb.Entry),
|
||||
}}
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) {
|
||||
if fake.txnCount != tc.markerTxn {
|
||||
return
|
||||
}
|
||||
fake.entries["/buckets/b/obj.versions/v_retry"] = proto.Clone(retryMarker).(*filer_pb.Entry)
|
||||
pointer := fake.entries["/buckets/b/obj.versions"]
|
||||
pointer.Extended[s3_constants.ExtLatestVersionIdKey] = []byte("retry")
|
||||
pointer.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte("v_retry")
|
||||
pointer.Extended[s3_constants.ExtLatestVersionIsDeleteMarker] = []byte("true")
|
||||
}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: tc.versioning})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: "old",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(latest),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.outcome, resp.Outcome, resp.Reason)
|
||||
var markers []string
|
||||
for key, e := range fake.entries {
|
||||
if string(e.Extended[s3_constants.ExtDeleteMarkerKey]) == "true" {
|
||||
markers = append(markers, key)
|
||||
}
|
||||
}
|
||||
require.Equal(t, []string{"/buckets/b/obj.versions/v_retry"}, markers)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A version promoted after the expiration checks passed must not be hidden by
|
||||
// the marker: the conflict retry finds a different latest and resolves stale.
|
||||
func TestLifecycleDelete_VersionPromotedBeforeMarkerIsNotHidden(t *testing.T) {
|
||||
latest := &filer_pb.Entry{
|
||||
Name: "v_old",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")},
|
||||
}
|
||||
promoted := &filer_pb.Entry{
|
||||
Name: "v_new",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("new")},
|
||||
}
|
||||
for _, versionId := range []string{"old", ""} {
|
||||
t.Run("version="+versionId, func(t *testing.T) {
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte("old"),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"),
|
||||
}},
|
||||
"/buckets/b/obj.versions/v_old": proto.Clone(latest).(*filer_pb.Entry),
|
||||
}}
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) {
|
||||
if fake.txnCount != 1 {
|
||||
return
|
||||
}
|
||||
fake.entries["/buckets/b/obj.versions/v_new"] = proto.Clone(promoted).(*filer_pb.Entry)
|
||||
pointer := fake.entries["/buckets/b/obj.versions"]
|
||||
pointer.Extended[s3_constants.ExtLatestVersionIdKey] = []byte("new")
|
||||
pointer.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte("v_new")
|
||||
}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: versionId,
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(latest),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED, resp.Outcome, resp.Reason)
|
||||
require.Equal(t, "VERSION_NOT_LATEST", resp.Reason)
|
||||
for key, e := range fake.entries {
|
||||
require.NotEqual(t, "true", string(e.Extended[s3_constants.ExtDeleteMarkerKey]), key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_VersionPromotedOverNullBeforeMarkerIsNotHidden(t *testing.T) {
|
||||
null := &filer_pb.Entry{
|
||||
Name: "obj",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
}
|
||||
promoted := &filer_pb.Entry{
|
||||
Name: "v_new",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("new")},
|
||||
}
|
||||
for _, versionId := range []string{"null", ""} {
|
||||
t.Run("version="+versionId, func(t *testing.T) {
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj": proto.Clone(null).(*filer_pb.Entry),
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtNullVersionIsLatestKey: []byte("true"),
|
||||
}},
|
||||
}}
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) {
|
||||
if fake.txnCount != 1 {
|
||||
return
|
||||
}
|
||||
fake.entries["/buckets/b/obj.versions/v_new"] = proto.Clone(promoted).(*filer_pb.Entry)
|
||||
pointer := fake.entries["/buckets/b/obj.versions"]
|
||||
delete(pointer.Extended, s3_constants.ExtNullVersionIsLatestKey)
|
||||
pointer.Extended[s3_constants.ExtLatestVersionIdKey] = []byte("new")
|
||||
pointer.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte("v_new")
|
||||
}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: versionId,
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED, resp.Outcome, resp.Reason)
|
||||
require.Equal(t, "VERSION_NOT_LATEST", resp.Reason)
|
||||
for key, e := range fake.entries {
|
||||
require.NotEqual(t, "true", string(e.Extended[s3_constants.ExtDeleteMarkerKey]), key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_SuspendedPutOverVersionBeforeMarkerIsNotHidden(t *testing.T) {
|
||||
old := &filer_pb.Entry{
|
||||
Name: "v_old",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")},
|
||||
}
|
||||
fresh := &filer_pb.Entry{
|
||||
Name: "obj",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}},
|
||||
}
|
||||
for _, versionId := range []string{"old", ""} {
|
||||
t.Run("version="+versionId, func(t *testing.T) {
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions/v_old": proto.Clone(old).(*filer_pb.Entry),
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte("old"),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"),
|
||||
}},
|
||||
}}
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) {
|
||||
if fake.txnCount != 1 {
|
||||
return
|
||||
}
|
||||
fake.entries["/buckets/b/obj"] = proto.Clone(fresh).(*filer_pb.Entry)
|
||||
pointer := fake.entries["/buckets/b/obj.versions"]
|
||||
delete(pointer.Extended, s3_constants.ExtLatestVersionIdKey)
|
||||
delete(pointer.Extended, s3_constants.ExtLatestVersionFileNameKey)
|
||||
pointer.Extended[s3_constants.ExtNullVersionIsLatestKey] = []byte("true")
|
||||
}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: versionId,
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(old),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED, resp.Outcome, resp.Reason)
|
||||
require.Equal(t, "VERSION_NOT_LATEST", resp.Reason)
|
||||
for key, e := range fake.entries {
|
||||
require.NotEqual(t, "true", string(e.Extended[s3_constants.ExtDeleteMarkerKey]), key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_NullUnderIdOnlyPointerExpires(t *testing.T) {
|
||||
null := &filer_pb.Entry{
|
||||
Name: "obj",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
}
|
||||
for _, versionId := range []string{"null", ""} {
|
||||
t.Run("version="+versionId, func(t *testing.T) {
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj": proto.Clone(null).(*filer_pb.Entry),
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte("old"),
|
||||
}},
|
||||
}}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: versionId,
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason)
|
||||
markers := 0
|
||||
for _, e := range fake.entries {
|
||||
if string(e.Extended[s3_constants.ExtDeleteMarkerKey]) == "true" {
|
||||
markers++
|
||||
}
|
||||
}
|
||||
require.Equal(t, 1, markers)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type releaseObjectWriteLock func()
|
||||
|
||||
func (r releaseObjectWriteLock) StopShortLivedLock() error {
|
||||
r()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Off the ring a PUT that commits between the marker's pointer read and its
|
||||
// write would be hidden; holding the object write lock makes it wait and land
|
||||
// after the marker instead.
|
||||
func TestLifecycleDelete_UnroutedMarkerHoldsObjectWriteLock(t *testing.T) {
|
||||
old := &filer_pb.Entry{
|
||||
Name: "v_old",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")},
|
||||
}
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions/v_old": proto.Clone(old).(*filer_pb.Entry),
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte("old"),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"),
|
||||
}},
|
||||
}}
|
||||
interleavedId := generateVersionId(true)
|
||||
put := func(id string) {
|
||||
fake.entries["/buckets/b/obj.versions/v_"+id] = &filer_pb.Entry{
|
||||
Name: "v_" + id,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte(id)},
|
||||
}
|
||||
pointer := fake.entries["/buckets/b/obj.versions"]
|
||||
pointer.Extended[s3_constants.ExtLatestVersionIdKey] = []byte(id)
|
||||
pointer.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte("v_" + id)
|
||||
delete(pointer.Extended, s3_constants.ExtLatestVersionIsDeleteMarker)
|
||||
}
|
||||
held, waiting, putDone := false, false, false
|
||||
fake.beforeCreate = func() {
|
||||
switch {
|
||||
case putDone || waiting:
|
||||
case held:
|
||||
waiting = true
|
||||
default:
|
||||
put(interleavedId)
|
||||
putDone = true
|
||||
}
|
||||
}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled})
|
||||
s3a.newObjectWriteLock = func(bucket, object string) (objectWriteLock, error) {
|
||||
held = true
|
||||
return releaseObjectWriteLock(func() {
|
||||
fake.mu.Lock()
|
||||
defer fake.mu.Unlock()
|
||||
held = false
|
||||
if waiting {
|
||||
put(generateVersionId(true))
|
||||
putDone = true
|
||||
}
|
||||
}), nil
|
||||
}
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: "old",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(old),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason)
|
||||
require.True(t, putDone)
|
||||
latest, err := s3a.getLatestObjectVersion("b", "obj")
|
||||
require.NoError(t, err)
|
||||
require.NotEqual(t, "true", string(latest.Extended[s3_constants.ExtDeleteMarkerKey]), "the PUT must not be hidden by the marker")
|
||||
}
|
||||
|
||||
// Expired-marker cleanup removes a marker that a lock was recorded on: markers
|
||||
// are never WORM-protected, so the routed delete carries no WORM guard.
|
||||
func TestLifecycleDelete_LockedExpiredDeleteMarkerSkipsWormGuard(t *testing.T) {
|
||||
@@ -552,3 +983,123 @@ func TestLifecycleDelete_LockedExpiredDeleteMarkerSkipsWormGuard(t *testing.T) {
|
||||
require.Len(t, conditions, 1)
|
||||
require.Nil(t, conditions[0], "marker delete must not carry WORM guards")
|
||||
}
|
||||
|
||||
// A worker retries a request whose reply it lost; the replay must resolve
|
||||
// against what the first attempt already did.
|
||||
func TestLifecycleDelete_ReplayAfterLostReply(t *testing.T) {
|
||||
version := func(id string, mtime int64, extra map[string][]byte) *filer_pb.Entry {
|
||||
e := &filer_pb.Entry{
|
||||
Name: "v_" + id,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: mtime, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,0" + id + "637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte(id)},
|
||||
}
|
||||
for k, v := range extra {
|
||||
e.Extended[k] = v
|
||||
}
|
||||
return e
|
||||
}
|
||||
versionsDir := func(latestId string) *filer_pb.Entry {
|
||||
return &filer_pb.Entry{
|
||||
Name: "obj" + s3_constants.VersionsFolder,
|
||||
IsDirectory: true,
|
||||
Attributes: &filer_pb.FuseAttributes{},
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte(latestId),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_" + latestId),
|
||||
},
|
||||
}
|
||||
}
|
||||
locked := map[string][]byte{
|
||||
s3_constants.ExtObjectLockModeKey: []byte(s3_constants.RetentionModeCompliance),
|
||||
s3_constants.ExtRetentionUntilDateKey: []byte(strconv.FormatInt(time.Now().Add(24*time.Hour).Unix(), 10)),
|
||||
}
|
||||
oldVersion := version("1", 1_700_000_000, nil)
|
||||
newVersion := version("2", 1_700_000_100, nil)
|
||||
lockedVersion := version("1", 1_700_000_000, locked)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
objectLock bool
|
||||
kind s3_lifecycle_pb.ActionKind
|
||||
target *filer_pb.Entry
|
||||
entries map[string]*filer_pb.Entry
|
||||
first s3_lifecycle_pb.LifecycleDeleteOutcome
|
||||
replay s3_lifecycle_pb.LifecycleDeleteOutcome
|
||||
}{
|
||||
{"current expiration writes one delete marker", false, s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, oldVersion, map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions": versionsDir("1"),
|
||||
"/buckets/b/obj.versions/v_1": oldVersion,
|
||||
}, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED},
|
||||
{"noncurrent version", false, s3_lifecycle_pb.ActionKind_NONCURRENT_DAYS, oldVersion, map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions": versionsDir("2"),
|
||||
"/buckets/b/obj.versions/v_1": oldVersion,
|
||||
"/buckets/b/obj.versions/v_2": newVersion,
|
||||
}, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED},
|
||||
{"object-locked noncurrent version", true, s3_lifecycle_pb.ActionKind_NONCURRENT_DAYS, lockedVersion, map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions": versionsDir("2"),
|
||||
"/buckets/b/obj.versions/v_1": lockedVersion,
|
||||
"/buckets/b/obj.versions/v_2": newVersion,
|
||||
}, s3_lifecycle_pb.LifecycleDeleteOutcome_SKIPPED_OBJECT_LOCK, s3_lifecycle_pb.LifecycleDeleteOutcome_SKIPPED_OBJECT_LOCK},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
entries := map[string]*filer_pb.Entry{}
|
||||
for k, v := range c.entries {
|
||||
entries[k] = proto.Clone(v).(*filer_pb.Entry)
|
||||
}
|
||||
fake := &ambiguousPutFiler{entries: entries, apply: true}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
cfg := &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled}
|
||||
if c.objectLock {
|
||||
cfg.ObjectLockConfig = &ObjectLockConfiguration{ObjectLockEnabled: s3_constants.ObjectLockEnabled}
|
||||
}
|
||||
s3a.bucketConfigCache.Set("b", cfg)
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
|
||||
req := &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: string(c.target.Extended[s3_constants.ExtVersionIdKey]),
|
||||
ActionKind: c.kind,
|
||||
ExpectedIdentity: computeEntryIdentity(c.target),
|
||||
}
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), req)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, c.first, resp.Outcome, resp.Reason)
|
||||
|
||||
fake.mu.Lock()
|
||||
afterFirst := len(fake.entries)
|
||||
txns := fake.txnCount
|
||||
fake.mu.Unlock()
|
||||
|
||||
resp, err = s3a.LifecycleDelete(s3LocalPeerCtx(), proto.Clone(req).(*s3_lifecycle_pb.LifecycleDeleteRequest))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, c.replay, resp.Outcome, resp.Reason)
|
||||
|
||||
fake.mu.Lock()
|
||||
defer fake.mu.Unlock()
|
||||
require.Equal(t, txns, fake.txnCount, "the replay must not write")
|
||||
require.Len(t, fake.entries, afterFirst)
|
||||
markers := 0
|
||||
for _, e := range fake.entries {
|
||||
if string(e.Extended[s3_constants.ExtDeleteMarkerKey]) == "true" {
|
||||
markers++
|
||||
}
|
||||
}
|
||||
if c.kind == s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS {
|
||||
require.Equal(t, 1, markers, "a replayed expiration must not stack delete markers")
|
||||
} else {
|
||||
require.Zero(t, markers)
|
||||
}
|
||||
if c.objectLock {
|
||||
require.Contains(t, fake.entries, "/buckets/b/obj.versions/v_1", "a locked version must survive both attempts")
|
||||
} else if c.kind == s3_lifecycle_pb.ActionKind_NONCURRENT_DAYS {
|
||||
require.NotContains(t, fake.entries, "/buckets/b/obj.versions/v_1")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -761,7 +761,7 @@ func (s3a *S3ApiServer) finalizeCopyDestination(dstBucket, dstObject, dstVersion
|
||||
// Route the pointer flip to the owner filer when known (off the
|
||||
// distributed lock); RECOMPUTE_LATEST picks the just-written version.
|
||||
if owner != "" {
|
||||
if code := s3a.routedVersionedFinalize(owner, dstBucket, normalizedObject, isNewFormatVersionId(versionId), nil); code != s3err.ErrNone {
|
||||
if code := s3a.routedVersionedFinalize(owner, dstBucket, normalizedObject, isNewFormatVersionId(versionId), nil, nil); code != s3err.ErrNone {
|
||||
if rollbackErr := s3a.rollbackCopyVersion(owner, dstBucket, normalizedObject, versionObjectPath); rollbackErr != nil {
|
||||
glog.Errorf("CopyObjectHandler: failed to rollback version %s for %s/%s after routed finalize error: %v", versionId, dstBucket, normalizedObject, rollbackErr)
|
||||
}
|
||||
|
||||
@@ -294,6 +294,8 @@ func (f *ambiguousPutFiler) ObjectTransaction(_ context.Context, req *filer_pb.O
|
||||
ok = current != nil && etagCandidateMatch(clause.Etags, md5Hex)
|
||||
case filer_pb.WriteCondition_IF_ETAG_NOT_MATCH:
|
||||
ok = current == nil || !etagCandidateMatch(clause.Etags, md5Hex)
|
||||
case filer_pb.WriteCondition_IF_ENTRY_EQUAL:
|
||||
ok = (current == nil) == (clause.ExpectedEntry == nil) && (current == nil || proto.Equal(current, clause.ExpectedEntry))
|
||||
}
|
||||
if !ok {
|
||||
return &filer_pb.ObjectTransactionResponse{ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil
|
||||
|
||||
@@ -74,8 +74,9 @@ func (s3a *S3ApiServer) latestPointerRecompute(bucket, object string, useInverte
|
||||
// owner filer, via a single RECOMPUTE_LATEST. versionEntry, when set, is PUT
|
||||
// into .versions in the same transaction, so the recompute's scan sees it even
|
||||
// when the gateway's own filer is not the owner; otherwise the version file is
|
||||
// already written on the owner.
|
||||
func (s3a *S3ApiServer) routedVersionedFinalize(owner pb.ServerAddress, bucket, object string, useInvertedFormat bool, versionEntry *filer_pb.Entry) s3err.ErrorCode {
|
||||
// already written on the owner. latestGuard, when set, must hold on the
|
||||
// .versions directory entry under the same lock.
|
||||
func (s3a *S3ApiServer) routedVersionedFinalize(owner pb.ServerAddress, bucket, object string, useInvertedFormat bool, versionEntry *filer_pb.Entry, latestGuard *filer_pb.WriteCondition) s3err.ErrorCode {
|
||||
mutations := []*filer_pb.ObjectMutation{s3a.latestPointerRecompute(bucket, object, useInvertedFormat, "", true)}
|
||||
if versionEntry != nil {
|
||||
mutations = append([]*filer_pb.ObjectMutation{{
|
||||
@@ -89,11 +90,17 @@ func (s3a *S3ApiServer) routedVersionedFinalize(owner pb.ServerAddress, bucket,
|
||||
RouteKey: s3a.objectRouteKey(bucket, object),
|
||||
Mutations: mutations,
|
||||
}
|
||||
if latestGuard != nil {
|
||||
req.Condition = latestGuard
|
||||
req.ConditionKey = s3a.toFilerPath(bucket, object+s3_constants.VersionsFolder)
|
||||
}
|
||||
resp, err := s3a.objectTxnOnFiler(owner, req)
|
||||
switch {
|
||||
case err != nil:
|
||||
glog.Errorf("routedVersionedFinalize: %s/%s on %s: %v", bucket, object, owner, err)
|
||||
return s3err.ErrInternalError
|
||||
case resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED:
|
||||
return s3err.ErrPreconditionFailed
|
||||
case resp.Error != "":
|
||||
glog.Errorf("routedVersionedFinalize: %s/%s: %s", bucket, object, resp.Error)
|
||||
return s3err.ErrInternalError
|
||||
|
||||
@@ -201,8 +201,18 @@ type ObjectVersion struct {
|
||||
StorageClass string
|
||||
}
|
||||
|
||||
// errLatestVersionChanged reports a delete marker refused by its latestGuard.
|
||||
var errLatestVersionChanged = errors.New("latest version changed")
|
||||
|
||||
// createDeleteMarker creates a delete marker for versioned delete operations
|
||||
func (s3a *S3ApiServer) createDeleteMarker(bucket, object string) (string, error) {
|
||||
return s3a.createDeleteMarkerIf(bucket, object, nil)
|
||||
}
|
||||
|
||||
// createDeleteMarkerIf is createDeleteMarker whose routed commit also requires
|
||||
// latestGuard to hold on the .versions directory entry. The unrouted path
|
||||
// has no transaction to check it in and writes unconditionally.
|
||||
func (s3a *S3ApiServer) createDeleteMarkerIf(bucket, object string, latestGuard *filer_pb.WriteCondition) (string, error) {
|
||||
// Clean up the object path first
|
||||
cleanObject := strings.TrimPrefix(object, "/")
|
||||
|
||||
@@ -239,7 +249,11 @@ func (s3a *S3ApiServer) createDeleteMarker(bucket, object string) (string, error
|
||||
},
|
||||
Extended: deleteMarkerExtended,
|
||||
}
|
||||
if code := s3a.routedVersionedFinalize(owner, bucket, cleanObject, useInvertedFormat, deleteMarkerEntry); code != s3err.ErrNone {
|
||||
code := s3a.routedVersionedFinalize(owner, bucket, cleanObject, useInvertedFormat, deleteMarkerEntry, latestGuard)
|
||||
if code == s3err.ErrPreconditionFailed {
|
||||
return "", errLatestVersionChanged
|
||||
}
|
||||
if code != s3err.ErrNone {
|
||||
return "", fmt.Errorf("createDeleteMarker: routed finalize failed for %s/%s: code %d", bucket, object, code)
|
||||
}
|
||||
glog.V(2).Infof("createDeleteMarker: successfully created delete marker %s for %s/%s", versionId, bucket, object)
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package dailyrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/time/rate"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
func TestDispatch_UnavailableThenSuccess(t *testing.T) {
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{
|
||||
{err: status.Error(codes.Unavailable, "connection error")},
|
||||
{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
|
||||
}}
|
||||
out, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), 0)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, out)
|
||||
assert.Equal(t, int32(2), c.calls)
|
||||
}
|
||||
|
||||
func TestDispatch_ServerDeadlineIsRetried(t *testing.T) {
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{
|
||||
{err: status.Error(codes.DeadlineExceeded, "context deadline exceeded")},
|
||||
{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
|
||||
}}
|
||||
out, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), time.Minute)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, out)
|
||||
assert.Equal(t, int32(2), c.calls)
|
||||
}
|
||||
|
||||
func TestDispatch_PermanentErrorsAreNotRetried(t *testing.T) {
|
||||
for _, code := range []codes.Code{codes.Unauthenticated, codes.PermissionDenied, codes.InvalidArgument, codes.Unimplemented} {
|
||||
t.Run(code.String(), func(t *testing.T) {
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{{err: status.Error(code, "rejected")}}}
|
||||
_, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), 0)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, code, status.Code(err))
|
||||
assert.Equal(t, int32(1), c.calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatch_NilResponseIsAnErrorNotAPanic(t *testing.T) {
|
||||
c := &walkerStubClient{nilResp: true}
|
||||
_, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), 0)
|
||||
require.ErrorIs(t, err, errNilDeleteResponse)
|
||||
}
|
||||
|
||||
func TestDispatch_EveryAttemptWaitsOnLimiter(t *testing.T) {
|
||||
lim := rate.NewLimiter(rate.Every(time.Hour), transportRetryAttempts)
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{
|
||||
{err: status.Error(codes.Unavailable, "down")},
|
||||
{err: status.Error(codes.Unavailable, "down")},
|
||||
{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
|
||||
}}
|
||||
_, err := dispatchWithRetry(context.Background(), c, lim, sampleMatch(), 0)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int32(transportRetryAttempts), c.calls)
|
||||
assert.Less(t, lim.Tokens(), 0.5, "each attempt must take a token from the shared budget")
|
||||
}
|
||||
|
||||
func TestProcessMatches_RetryExhaustionHoldsCursorBeforeTheMatch(t *testing.T) {
|
||||
runNow := time.Now()
|
||||
first, second := sampleMatch(), sampleMatch()
|
||||
first.DueTime, second.DueTime = runNow.Add(-time.Hour), runNow.Add(-time.Hour)
|
||||
second.ObjectKey = "obj2"
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{
|
||||
{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
|
||||
{err: status.Error(codes.Unavailable, "down")},
|
||||
}}
|
||||
_, halted, err := processMatches(context.Background(), Config{Client: c}, runNow, &reader.Event{TsNs: runNow.UnixNano()}, []router.Match{first, second})
|
||||
require.Error(t, err)
|
||||
assert.True(t, halted)
|
||||
assert.Equal(t, int32(1+transportRetryAttempts), c.calls)
|
||||
}
|
||||
|
||||
func TestWalkerDispatcher_TransientErrorIsRetried(t *testing.T) {
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{
|
||||
{err: status.Error(codes.Unavailable, "connection reset")},
|
||||
{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
|
||||
}}
|
||||
d := &WalkerDispatcher{Client: c}
|
||||
err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int32(2), c.calls)
|
||||
}
|
||||
|
||||
func TestWalkerDispatcher_AuthErrorSurfacesAtOnce(t *testing.T) {
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{{err: status.Error(codes.Unauthenticated, "invalid admin token")}}}
|
||||
d := &WalkerDispatcher{Client: c}
|
||||
err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, codes.Unauthenticated, status.Code(err))
|
||||
assert.Equal(t, int32(1), c.calls)
|
||||
}
|
||||
|
||||
func TestWalkerDispatcher_ParentCancelDuringBackoffStops(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{{err: errors.New("transport: connection refused")}}}
|
||||
time.AfterFunc(10*time.Millisecond, cancel)
|
||||
d := &WalkerDispatcher{Client: c}
|
||||
err := d.Delete(ctx, sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.Equal(t, int32(1), c.calls)
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package dailyrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/time/rate"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
// stallingClient blocks each attempt until its context ends, except
|
||||
// attempts listed in answerAfter, which answer DONE after that delay.
|
||||
type stallingClient struct {
|
||||
mu sync.Mutex
|
||||
requests []*s3_lifecycle_pb.LifecycleDeleteRequest
|
||||
answerAfter map[int]time.Duration
|
||||
onCall func(attempt int)
|
||||
}
|
||||
|
||||
func (c *stallingClient) LifecycleDelete(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
|
||||
c.mu.Lock()
|
||||
c.requests = append(c.requests, req)
|
||||
attempt := len(c.requests)
|
||||
delay, answers := c.answerAfter[attempt]
|
||||
c.mu.Unlock()
|
||||
if c.onCall != nil {
|
||||
c.onCall(attempt)
|
||||
}
|
||||
if answers {
|
||||
select {
|
||||
case <-time.After(delay):
|
||||
return &s3_lifecycle_pb.LifecycleDeleteResponse{Outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
func (c *stallingClient) calls() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return len(c.requests)
|
||||
}
|
||||
|
||||
func versionedMatch() router.Match {
|
||||
m := sampleMatch()
|
||||
m.Key.ActionKind = s3lifecycle.ActionKindNoncurrentDays
|
||||
m.VersionID = "v_1"
|
||||
m.Identity = &router.EntryIdentity{MtimeNs: 1, Size: 7, HeadFid: "3,01", ExtendedHash: []byte{0xaa}}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestDispatch_StalledAttemptsTimeOutAndFail(t *testing.T) {
|
||||
c := &stallingClient{}
|
||||
start := time.Now()
|
||||
_, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), 20*time.Millisecond)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
assert.Contains(t, err.Error(), "timed out after 20ms")
|
||||
assert.Equal(t, transportRetryAttempts, c.calls(), "an attempt timeout is retried like a transport error")
|
||||
assert.Less(t, time.Since(start), 5*time.Second)
|
||||
}
|
||||
|
||||
func TestDispatch_SlowAttemptWithinTimeoutSucceeds(t *testing.T) {
|
||||
c := &stallingClient{answerAfter: map[int]time.Duration{1: 30 * time.Millisecond}}
|
||||
out, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), time.Second)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, out)
|
||||
assert.Equal(t, 1, c.calls())
|
||||
}
|
||||
|
||||
func TestDispatch_RetryAfterTimeoutResendsSameRequest(t *testing.T) {
|
||||
c := &stallingClient{answerAfter: map[int]time.Duration{2: 0}}
|
||||
out, err := dispatchWithRetry(context.Background(), c, nil, versionedMatch(), 20*time.Millisecond)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, out)
|
||||
require.Len(t, c.requests, 2)
|
||||
assert.True(t, proto.Equal(c.requests[0], c.requests[1]), "a retry after a lost reply must keep the version and identity witness")
|
||||
assert.Equal(t, "v_1", c.requests[1].VersionId)
|
||||
assert.Equal(t, "3,01", c.requests[1].ExpectedIdentity.GetHeadFid())
|
||||
}
|
||||
|
||||
func TestDispatch_ParentCancelInterruptsActiveAttempt(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c := &stallingClient{onCall: func(int) { time.AfterFunc(20*time.Millisecond, cancel) }}
|
||||
start := time.Now()
|
||||
_, err := dispatchWithRetry(ctx, c, nil, sampleMatch(), time.Hour)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.Equal(t, 1, c.calls(), "parent cancellation must not be retried")
|
||||
assert.Less(t, time.Since(start), 5*time.Second)
|
||||
}
|
||||
|
||||
func TestDispatch_ParentCancelInterruptsBackoff(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{{err: errors.New("transport boom")}}}
|
||||
time.AfterFunc(10*time.Millisecond, cancel)
|
||||
_, err := dispatchWithRetry(ctx, c, nil, sampleMatch(), time.Hour)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.Equal(t, int32(1), c.calls, "cancellation during backoff must stop before the next attempt")
|
||||
}
|
||||
|
||||
func TestProcessMatches_AttemptTimeoutFailsShard(t *testing.T) {
|
||||
runNow := time.Now()
|
||||
m := sampleMatch()
|
||||
m.DueTime = runNow.Add(-time.Hour)
|
||||
c := &stallingClient{}
|
||||
cfg := Config{Client: c, DeleteTimeout: 10 * time.Millisecond}
|
||||
_, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{TsNs: runNow.UnixNano()}, []router.Match{m})
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded, "exhausted attempts must fail the pass, not report it clean")
|
||||
assert.True(t, halted, "an unresolved delete must hold the cursor")
|
||||
}
|
||||
|
||||
func TestProcessMatches_LimiterWaitDoesNotSpendDeleteTimeout(t *testing.T) {
|
||||
runNow := time.Now()
|
||||
m := sampleMatch()
|
||||
m.DueTime = runNow.Add(-time.Hour)
|
||||
lim := rate.NewLimiter(rate.Every(150*time.Millisecond), 1)
|
||||
_ = lim.AllowN(time.Now(), 1)
|
||||
c := &stallingClient{answerAfter: map[int]time.Duration{1: 20 * time.Millisecond}}
|
||||
cfg := Config{Client: c, Limiter: lim, DeleteTimeout: 100 * time.Millisecond}
|
||||
_, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{TsNs: runNow.UnixNano()}, []router.Match{m})
|
||||
require.NoError(t, err)
|
||||
assert.False(t, halted)
|
||||
assert.Equal(t, 1, c.calls())
|
||||
}
|
||||
|
||||
func TestWalkerDispatcher_AttemptTimeoutHaltsWalk(t *testing.T) {
|
||||
c := &stallingClient{}
|
||||
d := &WalkerDispatcher{Client: c, DeleteTimeout: 10 * time.Millisecond}
|
||||
err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "timed out after 10ms")
|
||||
assert.Equal(t, transportRetryAttempts, c.calls(), "the walker shares the replay retry budget")
|
||||
}
|
||||
|
||||
func TestWalkerDispatcher_LimiterWaitDoesNotSpendDeleteTimeout(t *testing.T) {
|
||||
lim := rate.NewLimiter(rate.Every(150*time.Millisecond), 1)
|
||||
_ = lim.AllowN(time.Now(), 1)
|
||||
c := &stallingClient{answerAfter: map[int]time.Duration{1: 20 * time.Millisecond}}
|
||||
d := &WalkerDispatcher{Client: c, Limiter: lim, DeleteTimeout: 100 * time.Millisecond}
|
||||
err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -3,19 +3,22 @@ package dailyrun
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"golang.org/x/time/rate"
|
||||
"google.golang.org/grpc/codes"
|
||||
)
|
||||
|
||||
// transportRetryAttempts is the in-run retry budget for transport
|
||||
// errors (RPC dial failures, stream resets). Server-side outcomes
|
||||
// (RETRY_LATER, BLOCKED) are NOT retried in-run — the server already
|
||||
// classified them as "wait until next run." The retry exists only to
|
||||
// paper over a single flake from the network.
|
||||
// transportRetryAttempts is the in-run retry budget shared by the walker
|
||||
// and replay deletes. Only transport failures and attempt timeouts are
|
||||
// retried; server outcomes (RETRY_LATER, BLOCKED) wait for the next run.
|
||||
const transportRetryAttempts = 3
|
||||
|
||||
// transportRetryInitial is the first sleep after a failed attempt.
|
||||
@@ -26,41 +29,89 @@ const (
|
||||
transportRetryMax = 5 * time.Second
|
||||
)
|
||||
|
||||
// dispatchWithRetry sends one LifecycleDelete request, retrying on
|
||||
// transport errors up to transportRetryAttempts times with exponential
|
||||
// backoff. Returns the server's outcome on success, or an error if all
|
||||
// retries exhausted. Server outcomes (RETRY_LATER / BLOCKED) bypass
|
||||
// the retry — the daily run's halt-on-failure semantics handles them.
|
||||
func dispatchWithRetry(ctx context.Context, client LifecycleClient, m router.Match) (s3_lifecycle_pb.LifecycleDeleteOutcome, error) {
|
||||
req := buildDeleteRequest(m)
|
||||
// DefaultDeleteTimeout bounds one LifecycleDelete attempt when the
|
||||
// caller sets no timeout.
|
||||
const DefaultDeleteTimeout = time.Minute
|
||||
|
||||
var (
|
||||
errDeleteAttemptTimedOut = errors.New("lifecycle delete attempt timed out")
|
||||
errNilDeleteResponse = errors.New("nil response")
|
||||
)
|
||||
|
||||
// deleteOnce sends one LifecycleDelete attempt under its own deadline,
|
||||
// so a stalled RPC fails that attempt instead of holding the shard for
|
||||
// the job's lifetime.
|
||||
func deleteOnce(ctx context.Context, client LifecycleClient, req *s3_lifecycle_pb.LifecycleDeleteRequest, timeout time.Duration) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
|
||||
if timeout <= 0 {
|
||||
timeout = DefaultDeleteTimeout
|
||||
}
|
||||
attemptCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
resp, err := client.LifecycleDelete(attemptCtx, req)
|
||||
if err != nil && ctx.Err() == nil && attemptCtx.Err() != nil {
|
||||
err = fmt.Errorf("%w after %s: %w", errDeleteAttemptTimedOut, timeout, err)
|
||||
}
|
||||
if err == nil && resp == nil {
|
||||
err = errNilDeleteResponse
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// isRetryableDeleteError reports whether a failed attempt may be resent.
|
||||
// Authentication, permission and argument errors fail the same way
|
||||
// again, so they surface at once.
|
||||
func isRetryableDeleteError(err error) bool {
|
||||
if errors.Is(err, errDeleteAttemptTimedOut) || util.IsTransientError(err) {
|
||||
return true
|
||||
}
|
||||
st, ok := util.ServerStatus(err)
|
||||
return ok && st.Code() == codes.DeadlineExceeded
|
||||
}
|
||||
|
||||
// deleteWithRetry sends req, retrying transport failures and attempt
|
||||
// timeouts up to transportRetryAttempts times with jittered exponential
|
||||
// backoff. Every attempt waits on limiter, so retries stay inside the
|
||||
// shared delete budget. The resent request keeps its version and identity
|
||||
// witness, so a delete that committed before its reply was lost resolves
|
||||
// as NOOP_RESOLVED on the retry.
|
||||
func deleteWithRetry(ctx context.Context, client LifecycleClient, limiter *rate.Limiter, req *s3_lifecycle_pb.LifecycleDeleteRequest, timeout time.Duration) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
|
||||
backoff := transportRetryInitial
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= transportRetryAttempts; attempt++ {
|
||||
resp, err := client.LifecycleDelete(ctx, req)
|
||||
for attempt := 1; ; attempt++ {
|
||||
if limiter != nil {
|
||||
waitStart := time.Now()
|
||||
if err := limiter.Wait(ctx); err != nil {
|
||||
return nil, fmt.Errorf("limiter: %w", err)
|
||||
}
|
||||
stats.S3LifecycleDispatchLimiterWaitSeconds.Observe(time.Since(waitStart).Seconds())
|
||||
}
|
||||
resp, err := deleteOnce(ctx, client, req, timeout)
|
||||
if err == nil {
|
||||
return resp.Outcome, nil
|
||||
return resp, nil
|
||||
}
|
||||
// Context cancellation is shutdown, not a transport flake.
|
||||
// Surface it immediately so the caller halts and tomorrow
|
||||
// resumes from the same cursor.
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, err
|
||||
// Parent cancellation is shutdown, not a transport flake.
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return nil, ctxErr
|
||||
}
|
||||
lastErr = err
|
||||
if attempt == transportRetryAttempts {
|
||||
break
|
||||
if attempt >= transportRetryAttempts || !isRetryableDeleteError(err) {
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, ctx.Err()
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(jitter(backoff)):
|
||||
}
|
||||
backoff *= 2
|
||||
if backoff > transportRetryMax {
|
||||
backoff = transportRetryMax
|
||||
}
|
||||
backoff = min(backoff*2, transportRetryMax)
|
||||
}
|
||||
return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, lastErr
|
||||
}
|
||||
|
||||
// dispatchWithRetry sends the LifecycleDelete for m under the shared
|
||||
// retry policy and returns the server's outcome.
|
||||
func dispatchWithRetry(ctx context.Context, client LifecycleClient, limiter *rate.Limiter, m router.Match, timeout time.Duration) (s3_lifecycle_pb.LifecycleDeleteOutcome, error) {
|
||||
resp, err := deleteWithRetry(ctx, client, limiter, buildDeleteRequest(m), timeout)
|
||||
if err != nil {
|
||||
return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, err
|
||||
}
|
||||
return resp.Outcome, nil
|
||||
}
|
||||
|
||||
// jitter returns a duration in the range [d/2, d) using equal jitter.
|
||||
|
||||
@@ -56,7 +56,7 @@ func dispatchWithRetryFast(t *testing.T, ctx context.Context, c LifecycleClient,
|
||||
// We can't override the package-private constants, but every test
|
||||
// path here uses small attempt counts so even the production
|
||||
// backoff is fine.
|
||||
return dispatchWithRetry(ctx, c, m)
|
||||
return dispatchWithRetry(ctx, c, nil, m, 0)
|
||||
}
|
||||
|
||||
func TestDispatch_FirstAttemptSucceeds(t *testing.T) {
|
||||
|
||||
@@ -48,6 +48,10 @@ type Config struct {
|
||||
// nil -> no rate limit. Shared across all shard goroutines.
|
||||
Limiter *rate.Limiter
|
||||
|
||||
// DeleteTimeout bounds each LifecycleDelete attempt, applied after
|
||||
// the limiter wait. <= 0 uses DefaultDeleteTimeout.
|
||||
DeleteTimeout time.Duration
|
||||
|
||||
// Meta-log retention boundary. Rules whose effective TTL exceeds
|
||||
// this can't be serviced by replay alone and get partitioned into
|
||||
// the walk view (engine.PromotedHash). 0 falls back to maxTTL,
|
||||
@@ -675,8 +679,9 @@ func runShard(ctx context.Context, cfg Config, snap *engine.Snapshot, runNow tim
|
||||
if drainErr != nil {
|
||||
_ = saveCursorAndPublish(saveCtx, cfg.Persister, shardID, Cursor{TsNs: lastOK, RuleSetHash: rsh, PromotedHash: promoted, LastWalkedNs: lastWalkedNs})
|
||||
// Cut short by its wall-clock cap: truncated, not failed — the
|
||||
// cursor above carries progress forward.
|
||||
if errors.Is(drainErr, context.DeadlineExceeded) || errors.Is(drainErr, context.Canceled) {
|
||||
// cursor above carries progress forward. A timed-out delete
|
||||
// attempt also wraps DeadlineExceeded, so judge by ctx itself.
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("shard=%d: drain: %w", shardID, drainErr)
|
||||
@@ -807,14 +812,7 @@ func processMatches(ctx context.Context, cfg Config, runNow time.Time, ev *reade
|
||||
skippedAny = true
|
||||
continue
|
||||
}
|
||||
if cfg.Limiter != nil {
|
||||
waitStart := time.Now()
|
||||
if waitErr := cfg.Limiter.Wait(ctx); waitErr != nil {
|
||||
return skippedAny, true, waitErr
|
||||
}
|
||||
stats.S3LifecycleDispatchLimiterWaitSeconds.Observe(time.Since(waitStart).Seconds())
|
||||
}
|
||||
outcome, dispatchErr := dispatchWithRetry(ctx, cfg.Client, m)
|
||||
outcome, dispatchErr := dispatchWithRetry(ctx, cfg.Client, cfg.Limiter, m, cfg.DeleteTimeout)
|
||||
if dispatchErr != nil {
|
||||
glog.V(1).Infof("daily_run: transport error on %s/%s %s: %v",
|
||||
m.Bucket, m.ObjectKey, m.Key.ActionKind, dispatchErr)
|
||||
@@ -822,7 +820,7 @@ func processMatches(ctx context.Context, cfg Config, runNow time.Time, ev *reade
|
||||
// (dispatcher/dispatcher.go) so transport failures
|
||||
// aggregate under one outcome key across paths.
|
||||
countDispatch(cfg.Stats, m.Bucket, m.Key.ActionKind.String(), "RPC_ERROR")
|
||||
return skippedAny, true, nil
|
||||
return skippedAny, true, fmt.Errorf("%s/%s %s: %w", m.Bucket, m.ObjectKey, m.Key.ActionKind, dispatchErr)
|
||||
}
|
||||
countDispatch(cfg.Stats, m.Bucket, m.Key.ActionKind.String(), outcome.String())
|
||||
switch outcome {
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
@@ -24,6 +23,9 @@ type WalkerDispatcher struct {
|
||||
// daily-run's processMatches uses so the walker and replay paths
|
||||
// can't combine to burst past the cap. nil disables throttling.
|
||||
Limiter *rate.Limiter
|
||||
// DeleteTimeout bounds each LifecycleDelete, applied after the
|
||||
// limiter wait. <= 0 uses DefaultDeleteTimeout.
|
||||
DeleteTimeout time.Duration
|
||||
// nil skips per-pass outcome counts.
|
||||
Stats *RunStats
|
||||
}
|
||||
@@ -66,27 +68,13 @@ func (d *WalkerDispatcher) Delete(ctx context.Context, action *engine.CompiledAc
|
||||
ExpectedIdentity: toProtoIdentity(entry.Identity),
|
||||
}
|
||||
kindLabel := action.Key.ActionKind.String()
|
||||
if d.Limiter != nil {
|
||||
waitStart := time.Now()
|
||||
if waitErr := d.Limiter.Wait(ctx); waitErr != nil {
|
||||
return fmt.Errorf("walker dispatch %s/%s %s: limiter: %w", action.Bucket, objectPath, action.Key.ActionKind, waitErr)
|
||||
}
|
||||
stats.S3LifecycleDispatchLimiterWaitSeconds.Observe(time.Since(waitStart).Seconds())
|
||||
}
|
||||
resp, err := d.Client.LifecycleDelete(ctx, req)
|
||||
resp, err := deleteWithRetry(ctx, d.Client, d.Limiter, req, d.DeleteTimeout)
|
||||
if err != nil {
|
||||
// "RPC_ERROR" matches the streaming dispatcher and processMatches
|
||||
// labels so transport-class failures aggregate under one key.
|
||||
countDispatch(d.Stats, action.Bucket, kindLabel, "RPC_ERROR")
|
||||
return fmt.Errorf("walker dispatch %s/%s %s: %w", action.Bucket, objectPath, action.Key.ActionKind, err)
|
||||
}
|
||||
if resp == nil {
|
||||
// A misbehaving server stub returning (nil, nil) would panic on
|
||||
// the switch below. Bucket under RPC_ERROR rather than a new
|
||||
// label — operationally it's the same class of failure.
|
||||
countDispatch(d.Stats, action.Bucket, kindLabel, "RPC_ERROR")
|
||||
return fmt.Errorf("walker dispatch %s/%s %s: nil response", action.Bucket, objectPath, action.Key.ActionKind)
|
||||
}
|
||||
countDispatch(d.Stats, action.Bucket, kindLabel, resp.Outcome.String())
|
||||
switch resp.Outcome {
|
||||
case s3_lifecycle_pb.LifecycleDeleteOutcome_DONE,
|
||||
|
||||
@@ -42,6 +42,9 @@ const (
|
||||
// the per-shard walk budget — typically 60 (1h) for small clusters,
|
||||
// 360+ (6h+) for large ones.
|
||||
WalkerIntervalMinutesAdminKey = "walker_interval_minutes"
|
||||
// DeleteTimeoutSecondsAdminKey bounds each LifecycleDelete attempt.
|
||||
// 0 = dailyrun.DefaultDeleteTimeout.
|
||||
DeleteTimeoutSecondsAdminKey = "delete_timeout_seconds"
|
||||
|
||||
// MetadataKeyDeletesPerSecond is the per-worker share value the
|
||||
// admin writes into ClusterContext.Metadata at ExecuteJob time.
|
||||
|
||||
@@ -21,6 +21,9 @@ type Config struct {
|
||||
// behavior; positive values gate the walker via Cursor.LastWalkedNs
|
||||
// inside dailyrun.runShard.
|
||||
WalkerInterval time.Duration
|
||||
// DeleteTimeout bounds each LifecycleDelete attempt; 0 uses
|
||||
// dailyrun.DefaultDeleteTimeout.
|
||||
DeleteTimeout time.Duration
|
||||
}
|
||||
|
||||
func ParseConfig(adminValues map[string]*plugin_pb.ConfigValue, workerValues map[string]*plugin_pb.ConfigValue) Config {
|
||||
@@ -42,6 +45,9 @@ func ParseConfig(adminValues map[string]*plugin_pb.ConfigValue, workerValues map
|
||||
if mins := readInt64(adminValues, WalkerIntervalMinutesAdminKey, 0); mins > 0 {
|
||||
cfg.WalkerInterval = time.Duration(mins) * time.Minute
|
||||
}
|
||||
if secs := readInt64(adminValues, DeleteTimeoutSecondsAdminKey, 0); secs > 0 {
|
||||
cfg.DeleteTimeout = time.Duration(secs) * time.Second
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
|
||||
@@ -93,3 +93,21 @@ func TestParseConfigWalkerIntervalNegativeStaysZero(t *testing.T) {
|
||||
t.Errorf("negative WalkerInterval should stay 0, got %v", cfg.WalkerInterval)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseConfig_DeleteTimeout(t *testing.T) {
|
||||
if cfg := ParseConfig(nil, nil); cfg.DeleteTimeout != 0 {
|
||||
t.Errorf("DeleteTimeout default=%v, want 0 (dailyrun default)", cfg.DeleteTimeout)
|
||||
}
|
||||
cfg := ParseConfig(map[string]*plugin_pb.ConfigValue{
|
||||
DeleteTimeoutSecondsAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 45}},
|
||||
}, nil)
|
||||
if want := 45 * time.Second; cfg.DeleteTimeout != want {
|
||||
t.Errorf("DeleteTimeout=%v, want %v", cfg.DeleteTimeout, want)
|
||||
}
|
||||
cfg = ParseConfig(map[string]*plugin_pb.ConfigValue{
|
||||
DeleteTimeoutSecondsAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: -5}},
|
||||
}, nil)
|
||||
if cfg.DeleteTimeout != 0 {
|
||||
t.Errorf("negative DeleteTimeout should stay 0, got %v", cfg.DeleteTimeout)
|
||||
}
|
||||
}
|
||||
@@ -111,6 +111,14 @@ func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor {
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
},
|
||||
{
|
||||
Name: DeleteTimeoutSecondsAdminKey,
|
||||
Label: "Delete RPC Timeout (seconds)",
|
||||
Description: "Deadline for each lifecycle delete RPC to an S3 server, counted after the rate-limit wait. Replay and walker deletes get up to 3 attempts. After that a replay shard stops at that event and resumes there on the next run, and a walk fails and runs again from the start on the next run. 0 = 60 seconds.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -119,6 +127,7 @@ func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor {
|
||||
ClusterDeletesBurstAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
MetaLogRetentionDaysAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
WalkerIntervalMinutesAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
DeleteTimeoutSecondsAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
},
|
||||
},
|
||||
// WorkerConfigForm intentionally absent — the prior
|
||||
@@ -336,7 +345,7 @@ func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.Exe
|
||||
runStats := &dailyrun.RunStats{}
|
||||
// Share the limiter with processMatches so walker + replay can't
|
||||
// combine to burst past the cluster cap.
|
||||
walkerDispatch := &dailyrun.WalkerDispatcher{Client: client, Limiter: limiter, Stats: runStats}
|
||||
walkerDispatch := &dailyrun.WalkerDispatcher{Client: client, Limiter: limiter, DeleteTimeout: cfg.DeleteTimeout, Stats: runStats}
|
||||
walker := dailyrun.WalkerFunc(func(walkCtx context.Context, view *engine.Snapshot, shards []int) (time.Time, error) {
|
||||
return dailyrun.WalkBuckets(walkCtx, view, shards, buckets,
|
||||
dailyrun.FilerListFunc(runStats.CountListed(filerClient), bucketsPath, view), walkerDispatch, runStats, persister)
|
||||
@@ -360,6 +369,7 @@ func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.Exe
|
||||
Lister: dispatcher.NewFilerSiblingLister(filerClient, bucketsPath),
|
||||
Workers: cfg.Workers,
|
||||
Limiter: limiter,
|
||||
DeleteTimeout: cfg.DeleteTimeout,
|
||||
RetentionWindow: cfg.MetaLogRetention,
|
||||
Walker: walker,
|
||||
WalkerInterval: cfg.WalkerInterval,
|
||||
|
||||
Reference in new issue
Block a user