mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 00:38:53 +01:00
s3 lifecycle: write the suspended expiration marker before deleting the null version (#12664)
* s3 test: evaluate mutation conditions and serve DeleteEntry in the fake filer The fake ObjectTransaction now checks each mutation's own condition before applying it, as the filer does, and reports the failed mutation. ETag clauses read the stored ETag attribute first, like the filer. DeleteEntry removes the entry so unrouted delete paths can run against the fake. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: write the suspended expiration marker before deleting the null version A suspended-versioning expiration deleted the null version first and then wrote the delete marker. When the pointer read or the marker write failed in between, the retry found no null version, resolved as NOT_FOUND or VERSION_NOT_LATEST, and never wrote the marker, so an older version became the latest. The marker now goes first. The latest pointer outranks the null object, so the key reads as deleted once the marker lands, and a null delete that fails after it leaves a noncurrent null version rather than exposing an older one. The null delete is pinned to the entry read before the marker: on the owner it carries IF_ENTRY_EQUAL on that read, and off the ring the entry is re-read and compared by identity first. A null object written after the read is a newer write and is no longer deleted. When the null version is the one that passed the expiration checks, a missing or changed entry at that read resolves the request as STALE_IDENTITY before anything is written. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: retry a failed routed null delete instead of deleting on the default filer A routed null delete that failed fell through to the unrouted path, which reads and deletes on the default filer. That filer may not have the owner's newer null object yet, so the fallback could delete it without the owner-side IF_ENTRY_EQUAL guard. The failure now returns RETRY_LATER; the marker is already written, so the key keeps reading as deleted. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: recheck and delete an unrouted null version under the object write lock Without a ring owner the identity recheck and the null delete were separate calls, so a suspended PUT landing between them was deleted. Off the ring writers take the object write lock, so the recheck and the delete now run under it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 test: strip quotes from the stored ETag in the fake filer The filer normalizes both sides of an ETag comparison; the fake now strips quotes from the stored ETag attribute as well as from the candidate. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: retry the null delete when the object lock lookup fails on a routed key Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: check the suspended marker against the validated version A suspended expiration that validated a named version wrote its marker without comparing the pointer, so a null PUT landing after the checks was hidden, and deleted too when it landed before the null read. The marker now compares the latest the way the enabled path does and resolves the request as VERSION_NOT_LATEST. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 test: count the marker's object write lock in the unrouted null delete Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: recheck the attempt deadline after reading the null version Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c09009a74f
commit
5f7fefadae
3 files changed
+421
-69
No files matched your search
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
// LifecycleDelete executes one (rule, action) verdict: re-fetch, identity
|
||||
@@ -104,39 +105,24 @@ func (s3a *S3ApiServer) lifecycleDispatch(ctx context.Context, req *s3_lifecycle
|
||||
}
|
||||
return done(), nil
|
||||
case s3_constants.VersioningSuspended:
|
||||
// Best-effort null delete; NotFound is benign. Route it when an
|
||||
// owner resolves — on the default filer an absent entry deletes as
|
||||
// success, and a null version that only reached the owner would
|
||||
// linger. Lifecycle never bypasses retention.
|
||||
nullDeleted := false
|
||||
if owner := s3a.routableWriteOwner(req.Bucket, req.ObjectPath); owner != "" {
|
||||
worm, lockErr := s3a.isObjectLockEnabled(req.Bucket)
|
||||
if lockErr == nil {
|
||||
code, settled := s3a.routedDeleteNullVersion(owner, req.Bucket, req.ObjectPath, worm, false, metadataOnly, nil)
|
||||
if settled {
|
||||
if code == s3err.ErrAccessDenied {
|
||||
return retryLater("ACCESS_DENIED: deleteNullVersion: object lock"), nil
|
||||
}
|
||||
if code != s3err.ErrNone {
|
||||
return retryLater("TRANSPORT_ERROR: routedDeleteNullVersion: " + s3err.GetAPIError(code).Code), nil
|
||||
}
|
||||
nullDeleted = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !nullDeleted {
|
||||
if err := s3a.deleteSpecificObjectVersion(ctx, req.Bucket, req.ObjectPath, "null", metadataOnly); err != nil {
|
||||
if !errors.Is(err, filer_pb.ErrNotFound) && !errors.Is(err, ErrVersionNotFound) {
|
||||
return retryLater("TRANSPORT_ERROR: deleteNullVersion: " + err.Error()), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
// The null delete is done, so the marker must follow even past
|
||||
// the deadline: without it a retry finds an older version latest.
|
||||
if _, resp := s3a.lifecycleDeleteMarker(req.Bucket, req.ObjectPath, ""); resp != nil {
|
||||
// Marker first: the pointer outranks the null object, so the key
|
||||
// reads as deleted once it lands, and a null delete that fails
|
||||
// after it leaves a noncurrent version instead of exposing an
|
||||
// older one to a retry that then finds no null to expire.
|
||||
nullEntry, resp := s3a.lifecycleNullVersion(req, entry)
|
||||
if resp != nil {
|
||||
return resp, nil
|
||||
}
|
||||
return done(), nil
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, resp := s3a.lifecycleDeleteMarker(req.Bucket, req.ObjectPath, lifecycleAddressedVersion(req, entry)); resp != nil {
|
||||
return resp, nil
|
||||
}
|
||||
if nullEntry == nil {
|
||||
return done(), nil
|
||||
}
|
||||
return s3a.lifecycleDeleteNullVersion(req, nullEntry, metadataOnly), nil
|
||||
default:
|
||||
err := s3a.WithFilerClient(false, func(c filer_pb.SeaweedFilerClient) error {
|
||||
return s3a.deleteUnversionedObjectWithClient(ctx, c, req.Bucket, req.ObjectPath, metadataOnly)
|
||||
@@ -291,6 +277,77 @@ func (s3a *S3ApiServer) lifecycleDeleteMarkerIf(bucket, object, normalized, expe
|
||||
return false, retryLater("LATEST_CHANGED: createDeleteMarker")
|
||||
}
|
||||
|
||||
// lifecycleNullVersion reads the null object the suspended expiration will
|
||||
// delete. When it is the version validated for expiration, a different or
|
||||
// missing entry resolves the request as stale.
|
||||
func (s3a *S3ApiServer) lifecycleNullVersion(req *s3_lifecycle_pb.LifecycleDeleteRequest, validated *filer_pb.Entry) (*filer_pb.Entry, *s3_lifecycle_pb.LifecycleDeleteResponse) {
|
||||
nullEntry, err := s3a.getObjectEntryRoutedByKey(req.Bucket, req.ObjectPath)
|
||||
if err != nil && !isObjectEntryMiss(err) {
|
||||
return nil, retryLater("TRANSPORT_ERROR: null version lookup: " + err.Error())
|
||||
}
|
||||
if err != nil || !isNullVersionObject(nullEntry, req.ObjectPath) {
|
||||
nullEntry = nil
|
||||
}
|
||||
if lifecycleAddressedVersion(req, validated) == "null" && (nullEntry == nil || !identityMatches(computeEntryIdentity(nullEntry), computeEntryIdentity(validated))) {
|
||||
return nil, noopResolved("STALE_IDENTITY")
|
||||
}
|
||||
return nullEntry, nil
|
||||
}
|
||||
|
||||
// lifecycleDeleteNullVersion deletes the null version once the marker hides
|
||||
// it, even past the attempt deadline. Only the entry read before the marker is
|
||||
// deleted: a null object written since is newer and stays. Lifecycle never
|
||||
// bypasses retention.
|
||||
func (s3a *S3ApiServer) lifecycleDeleteNullVersion(req *s3_lifecycle_pb.LifecycleDeleteRequest, nullEntry *filer_pb.Entry, metadataOnly bool) *s3_lifecycle_pb.LifecycleDeleteResponse {
|
||||
if owner := s3a.routableWriteOwner(req.Bucket, req.ObjectPath); owner != "" {
|
||||
worm, lockErr := s3a.isObjectLockEnabled(req.Bucket)
|
||||
if lockErr != nil {
|
||||
return retryLater("TRANSPORT_ERROR: object lock lookup: " + lockErr.Error())
|
||||
}
|
||||
guard := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{
|
||||
Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL,
|
||||
ExpectedEntry: nullEntry,
|
||||
}}}
|
||||
switch code, _ := s3a.routedDeleteNullVersion(owner, req.Bucket, req.ObjectPath, worm, false, metadataOnly, guard); code {
|
||||
case s3err.ErrNone, s3err.ErrPreconditionFailed:
|
||||
return done()
|
||||
case s3err.ErrAccessDenied:
|
||||
return retryLater("ACCESS_DENIED: deleteNullVersion: object lock")
|
||||
default:
|
||||
return retryLater("TRANSPORT_ERROR: routedDeleteNullVersion: " + s3err.GetAPIError(code).Code)
|
||||
}
|
||||
}
|
||||
// Off the ring, writers serialize on the object write lock instead.
|
||||
dir, name := util.NewFullPath(s3a.bucketDir(req.Bucket), req.ObjectPath).DirAndName()
|
||||
var opErr error
|
||||
code := s3a.withObjectWriteLock(req.Bucket, req.ObjectPath, func() s3err.ErrorCode {
|
||||
current, err := s3a.getEntry(dir, name)
|
||||
switch {
|
||||
case errors.Is(err, filer_pb.ErrNotFound):
|
||||
return s3err.ErrPreconditionFailed
|
||||
case err != nil:
|
||||
opErr = err
|
||||
return s3err.ErrInternalError
|
||||
case !identityMatches(computeEntryIdentity(current), computeEntryIdentity(nullEntry)):
|
||||
return s3err.ErrPreconditionFailed
|
||||
}
|
||||
return s3err.ErrNone
|
||||
}, func() s3err.ErrorCode {
|
||||
if opErr = s3a.deleteSpecificObjectVersion(context.Background(), req.Bucket, req.ObjectPath, "null", metadataOnly); opErr != nil {
|
||||
return s3err.ErrInternalError
|
||||
}
|
||||
return s3err.ErrNone
|
||||
})
|
||||
switch {
|
||||
case code == s3err.ErrNone || code == s3err.ErrPreconditionFailed:
|
||||
return done()
|
||||
case opErr != nil:
|
||||
return retryLater("TRANSPORT_ERROR: deleteNullVersion: " + opErr.Error())
|
||||
default:
|
||||
return retryLater("TRANSPORT_ERROR: deleteNullVersion: " + s3err.GetAPIError(code).Code)
|
||||
}
|
||||
}
|
||||
|
||||
// pointerLatestVersion reads the latest version off a .versions pointer the
|
||||
// way getLatestObjectVersion does: a named pointer with its file outranks the
|
||||
// null object, the null-is-latest signal names the null version, and anything
|
||||
|
||||
@@ -3,6 +3,8 @@ package s3api
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"testing"
|
||||
@@ -540,7 +542,9 @@ func TestLifecycleDelete_ExpiredAttemptWritesNoMarker(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_DeadlineDuringNullDeleteStillWritesMarker(t *testing.T) {
|
||||
// suspendedNullServer serves a suspended bucket whose latest is the null
|
||||
// object, with an older version that a lost delete marker would expose.
|
||||
func suspendedNullServer(t *testing.T, routed bool) (*S3ApiServer, *ambiguousPutFiler, *filer_pb.Entry) {
|
||||
olderId := generateVersionId(true)
|
||||
older := &filer_pb.Entry{
|
||||
Name: "v_" + olderId,
|
||||
@@ -548,38 +552,290 @@ func TestLifecycleDelete_DeadlineDuringNullDeleteStillWritesMarker(t *testing.T)
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte(olderId)},
|
||||
}
|
||||
latest := &filer_pb.Entry{
|
||||
null := &filer_pb.Entry{
|
||||
Name: "obj",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("null")},
|
||||
}
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj": proto.Clone(latest).(*filer_pb.Entry),
|
||||
"/buckets/b/obj": proto.Clone(null).(*filer_pb.Entry),
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
|
||||
"/buckets/b/obj.versions/" + older.Name: proto.Clone(older).(*filer_pb.Entry),
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(s3LocalPeerCtx())
|
||||
defer cancel()
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { cancel() }
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningSuspended})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
if routed {
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
}
|
||||
return s3a, fake, null
|
||||
}
|
||||
|
||||
func isMarkerTxn(req *filer_pb.ObjectTransactionRequest) bool {
|
||||
for _, m := range req.Mutations {
|
||||
if m.Type == filer_pb.ObjectMutation_PUT {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func requireLatestIsMarker(t *testing.T, s3a *S3ApiServer) {
|
||||
t.Helper()
|
||||
got, err := s3a.getLatestObjectVersion("b", "obj")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "true", string(got.Extended[s3_constants.ExtDeleteMarkerKey]), "the older version must stay hidden behind the marker")
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_DeadlineDuringMarkerStillDeletesNull(t *testing.T) {
|
||||
s3a, fake, null := suspendedNullServer(t, true)
|
||||
ctx, cancel := context.WithCancel(s3LocalPeerCtx())
|
||||
defer cancel()
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { cancel() }
|
||||
resp, err := s3a.LifecycleDelete(ctx, &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(latest),
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason)
|
||||
require.Equal(t, 2, fake.txnCount, "the null delete and the marker")
|
||||
got, err := s3a.getLatestObjectVersion("b", "obj")
|
||||
require.Equal(t, 2, fake.txnCount, "the marker and the null delete")
|
||||
require.NotContains(t, fake.entries, "/buckets/b/obj")
|
||||
requireLatestIsMarker(t, s3a)
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_DeadlineDuringNullReadWritesNoMarker(t *testing.T) {
|
||||
nullReads := func(cancelAt int, cancel func()) int {
|
||||
s3a, fake, null := suspendedNullServer(t, true)
|
||||
reads := 0
|
||||
fake.beforeLookup = func(key string) {
|
||||
if key != "/buckets/b/obj" || fake.txnCount > 0 {
|
||||
return
|
||||
}
|
||||
if reads++; reads == cancelAt {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
ctx, stop := context.WithCancel(s3LocalPeerCtx())
|
||||
defer stop()
|
||||
if cancel == nil {
|
||||
cancel = stop
|
||||
}
|
||||
_, err := s3a.LifecycleDelete(ctx, &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
})
|
||||
if cancelAt > 0 {
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
require.Zero(t, fake.txnCount, "an attempt past its deadline must not write a delete marker")
|
||||
require.Contains(t, fake.entries, "/buckets/b/obj")
|
||||
}
|
||||
return reads
|
||||
}
|
||||
last := nullReads(0, nil)
|
||||
require.Positive(t, last)
|
||||
nullReads(last, nil)
|
||||
}
|
||||
|
||||
// A marker write that fails leaves the null version in place, so the retry
|
||||
// still finds it latest and finishes the expiration.
|
||||
func TestLifecycleDelete_SuspendedMarkerFailureLeavesNullForRetry(t *testing.T) {
|
||||
for _, routed := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("routed=%v", routed), func(t *testing.T) {
|
||||
s3a, fake, null := suspendedNullServer(t, routed)
|
||||
failMarker := true
|
||||
fake.txnCheck = func(req *filer_pb.ObjectTransactionRequest) {
|
||||
fake.txnNoApply = failMarker && isMarkerTxn(req)
|
||||
fake.txnErr = nil
|
||||
if fake.txnNoApply {
|
||||
fake.txnErr = errors.New("owner unavailable")
|
||||
}
|
||||
}
|
||||
fake.createErr = errors.New("filer unavailable")
|
||||
req := &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
}
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), req)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_RETRY_LATER, resp.Outcome, resp.Reason)
|
||||
require.Contains(t, fake.entries, "/buckets/b/obj", "the null version must outlive a failed marker")
|
||||
|
||||
failMarker, fake.createErr = false, nil
|
||||
resp, err = s3a.LifecycleDelete(s3LocalPeerCtx(), req)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason)
|
||||
require.NotContains(t, fake.entries, "/buckets/b/obj")
|
||||
requireLatestIsMarker(t, s3a)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A routed null delete that fails retries rather than deleting through the
|
||||
// default filer, which may not have the owner's newer null object yet.
|
||||
func TestLifecycleDelete_RoutedNullDeleteFailureRetries(t *testing.T) {
|
||||
s3a, fake, null := suspendedNullServer(t, true)
|
||||
fake.txnCheck = func(req *filer_pb.ObjectTransactionRequest) {
|
||||
fake.txnNoApply = !isMarkerTxn(req)
|
||||
fake.txnErr = nil
|
||||
if fake.txnNoApply {
|
||||
fake.txnErr = errors.New("owner unavailable")
|
||||
}
|
||||
}
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "true", string(got.Extended[s3_constants.ExtDeleteMarkerKey]), "the older version must stay hidden behind the marker")
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_RETRY_LATER, resp.Outcome, resp.Reason)
|
||||
require.Contains(t, fake.entries, "/buckets/b/obj")
|
||||
requireLatestIsMarker(t, s3a)
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_RoutedNullDeleteObjectLockLookupFailureRetries(t *testing.T) {
|
||||
s3a, fake, null := suspendedNullServer(t, true)
|
||||
fake.txnCheck = func(req *filer_pb.ObjectTransactionRequest) {
|
||||
if isMarkerTxn(req) {
|
||||
s3a.bucketConfigCache.Remove("b")
|
||||
fake.lookupErr, fake.lookupFailKey = status.Error(codes.Unavailable, "filer unavailable"), "/buckets/b"
|
||||
}
|
||||
}
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_RETRY_LATER, resp.Outcome, resp.Reason)
|
||||
require.Equal(t, 1, fake.txnCount, "only the marker")
|
||||
require.Contains(t, fake.entries, "/buckets/b/obj")
|
||||
}
|
||||
|
||||
// A null object written after the expiration read the old one is not deleted.
|
||||
func TestLifecycleDelete_SuspendedNullRewrittenBeforeDeleteSurvives(t *testing.T) {
|
||||
for _, routed := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("routed=%v", routed), func(t *testing.T) {
|
||||
s3a, fake, null := suspendedNullServer(t, routed)
|
||||
rewritten := proto.Clone(null).(*filer_pb.Entry)
|
||||
rewritten.Attributes.Mtime++
|
||||
rewritten.Chunks = []*filer_pb.FileChunk{{FileId: "3,03637037d6", Size: 3}}
|
||||
rewrite := func() { fake.entries["/buckets/b/obj"] = proto.Clone(rewritten).(*filer_pb.Entry) }
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) {
|
||||
if fake.txnCount == 1 {
|
||||
rewrite()
|
||||
}
|
||||
}
|
||||
fake.beforeCreate = rewrite
|
||||
var locked []string
|
||||
s3a.newObjectWriteLock = func(bucket, object string) (objectWriteLock, error) {
|
||||
locked = append(locked, bucket+"/"+object)
|
||||
return noopObjectWriteLock{}, nil
|
||||
}
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason)
|
||||
require.Contains(t, fake.entries, "/buckets/b/obj")
|
||||
require.Equal(t, "3,03637037d6", fake.entries["/buckets/b/obj"].Chunks[0].GetFileIdString())
|
||||
if !routed {
|
||||
require.Equal(t, []string{"b/obj", "b/obj"}, locked, "the marker, then the recheck and the delete, must hold the object write lock")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A null PUT that lands after a suspended expiration validated a named version
|
||||
// is neither hidden nor deleted, whether it lands before the null read or
|
||||
// before the routed marker commit.
|
||||
func TestLifecycleDelete_SuspendedPutOverNamedVersionSurvives(t *testing.T) {
|
||||
old := &filer_pb.Entry{
|
||||
Name: "v_old",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")},
|
||||
}
|
||||
fresh := &filer_pb.Entry{
|
||||
Name: "obj",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("null")},
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
routed bool
|
||||
atRead bool
|
||||
}{
|
||||
{"routed/at-null-read", true, true},
|
||||
{"routed/at-marker", true, false},
|
||||
{"unrouted/at-null-read", false, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions/v_old": proto.Clone(old).(*filer_pb.Entry),
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte("old"),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"),
|
||||
}},
|
||||
}}
|
||||
put := func() {
|
||||
if _, ok := fake.entries["/buckets/b/obj"]; ok {
|
||||
return
|
||||
}
|
||||
fake.entries["/buckets/b/obj"] = proto.Clone(fresh).(*filer_pb.Entry)
|
||||
pointer := fake.entries["/buckets/b/obj.versions"]
|
||||
delete(pointer.Extended, s3_constants.ExtLatestVersionIdKey)
|
||||
delete(pointer.Extended, s3_constants.ExtLatestVersionFileNameKey)
|
||||
pointer.Extended[s3_constants.ExtNullVersionIsLatestKey] = []byte("true")
|
||||
}
|
||||
if tc.atRead {
|
||||
fake.beforeLookup = func(key string) {
|
||||
if key == "/buckets/b/obj" {
|
||||
put()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { put() }
|
||||
}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningSuspended})
|
||||
if tc.routed {
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
}
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: "old",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(old),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED, resp.Outcome, resp.Reason)
|
||||
require.Equal(t, "VERSION_NOT_LATEST", resp.Reason)
|
||||
require.Contains(t, fake.entries, "/buckets/b/obj")
|
||||
for key, e := range fake.entries {
|
||||
require.NotEqual(t, "true", string(e.Extended[s3_constants.ExtDeleteMarkerKey]), key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A timed-out attempt can still be running when its retry writes the marker;
|
||||
@@ -605,7 +861,7 @@ func TestLifecycleDelete_OverlappingAttemptWritesOneMarker(t *testing.T) {
|
||||
outcome s3_lifecycle_pb.LifecycleDeleteOutcome
|
||||
}{
|
||||
{s3_constants.VersioningEnabled, 1, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED},
|
||||
{s3_constants.VersioningSuspended, 2, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
|
||||
{s3_constants.VersioningSuspended, 1, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.versioning, func(t *testing.T) {
|
||||
|
||||
@@ -155,7 +155,7 @@ type ambiguousPutFiler struct {
|
||||
// beforeLookup runs inside LookupDirectoryEntry, letting a test delete a
|
||||
// just-committed entry before recovery reads it.
|
||||
beforeCreate func()
|
||||
beforeLookup func()
|
||||
beforeLookup func(key string)
|
||||
beforeAssign func()
|
||||
stores int
|
||||
conds []*filer_pb.WriteCondition
|
||||
@@ -232,11 +232,18 @@ func putConditionSatisfied(cond *filer_pb.WriteCondition, current *filer_pb.Entr
|
||||
return true
|
||||
}
|
||||
|
||||
func (f *ambiguousPutFiler) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.entries, req.Directory+"/"+req.Name)
|
||||
return &filer_pb.DeleteEntryResponse{}, nil
|
||||
}
|
||||
|
||||
func (f *ambiguousPutFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.beforeLookup != nil {
|
||||
f.beforeLookup()
|
||||
f.beforeLookup(req.Directory + "/" + req.Name)
|
||||
}
|
||||
if f.lookupErr != nil && req.Directory+"/"+req.Name == f.lookupFailKey {
|
||||
return nil, f.lookupErr
|
||||
@@ -279,30 +286,23 @@ func (f *ambiguousPutFiler) ObjectTransaction(_ context.Context, req *filer_pb.O
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, clause := range req.GetCondition().GetClauses() {
|
||||
var md5Hex string
|
||||
if current != nil && current.Attributes != nil {
|
||||
md5Hex = fmt.Sprintf("%x", current.Attributes.Md5)
|
||||
}
|
||||
ok := true
|
||||
switch clause.Kind {
|
||||
case filer_pb.WriteCondition_IF_NOT_EXISTS:
|
||||
ok = current == nil
|
||||
case filer_pb.WriteCondition_IF_EXISTS:
|
||||
ok = current != nil
|
||||
case filer_pb.WriteCondition_IF_ETAG_MATCH:
|
||||
ok = current != nil && etagCandidateMatch(clause.Etags, md5Hex)
|
||||
case filer_pb.WriteCondition_IF_ETAG_NOT_MATCH:
|
||||
ok = current == nil || !etagCandidateMatch(clause.Etags, md5Hex)
|
||||
case filer_pb.WriteCondition_IF_ENTRY_EQUAL:
|
||||
ok = (current == nil) == (clause.ExpectedEntry == nil) && (current == nil || proto.Equal(current, clause.ExpectedEntry))
|
||||
}
|
||||
if !ok {
|
||||
return &filer_pb.ObjectTransactionResponse{ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil
|
||||
}
|
||||
if !txnConditionHolds(req.GetCondition(), current) {
|
||||
return &filer_pb.ObjectTransactionResponse{ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil
|
||||
}
|
||||
if !f.txnNoApply {
|
||||
for _, m := range req.Mutations {
|
||||
for i, m := range req.Mutations {
|
||||
if m.Condition != nil {
|
||||
target := m.ConditionKey
|
||||
if target == "" && m.Type == filer_pb.ObjectMutation_PUT {
|
||||
target = m.Directory + "/" + m.Entry.Name
|
||||
} else if target == "" {
|
||||
target = m.Directory + "/" + m.Name
|
||||
}
|
||||
if !txnConditionHolds(m.Condition, f.entries[target]) {
|
||||
failed := int32(i)
|
||||
return &filer_pb.ObjectTransactionResponse{ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED, FailedMutation: &failed}, nil
|
||||
}
|
||||
}
|
||||
switch m.Type {
|
||||
case filer_pb.ObjectMutation_PUT:
|
||||
entry := proto.Clone(m.Entry).(*filer_pb.Entry)
|
||||
@@ -323,6 +323,36 @@ func (f *ambiguousPutFiler) ObjectTransaction(_ context.Context, req *filer_pb.O
|
||||
return &filer_pb.ObjectTransactionResponse{}, f.txnErr
|
||||
}
|
||||
|
||||
// txnConditionHolds evaluates an ObjectTransaction condition, request-level or
|
||||
// on one mutation, the way the filer does for the kinds the tests send.
|
||||
func txnConditionHolds(cond *filer_pb.WriteCondition, current *filer_pb.Entry) bool {
|
||||
for _, clause := range cond.GetClauses() {
|
||||
var etag string
|
||||
if v := current.GetExtended()[s3_constants.ExtETagKey]; len(v) > 0 {
|
||||
etag = strings.Trim(string(v), `"`)
|
||||
} else if current != nil && current.Attributes != nil {
|
||||
etag = fmt.Sprintf("%x", current.Attributes.Md5)
|
||||
}
|
||||
ok := true
|
||||
switch clause.Kind {
|
||||
case filer_pb.WriteCondition_IF_NOT_EXISTS:
|
||||
ok = current == nil
|
||||
case filer_pb.WriteCondition_IF_EXISTS:
|
||||
ok = current != nil
|
||||
case filer_pb.WriteCondition_IF_ETAG_MATCH:
|
||||
ok = current != nil && etagCandidateMatch(clause.Etags, etag)
|
||||
case filer_pb.WriteCondition_IF_ETAG_NOT_MATCH:
|
||||
ok = current == nil || !etagCandidateMatch(clause.Etags, etag)
|
||||
case filer_pb.WriteCondition_IF_ENTRY_EQUAL:
|
||||
ok = (current == nil) == (clause.ExpectedEntry == nil) && (current == nil || proto.Equal(current, clause.ExpectedEntry))
|
||||
}
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// etagCandidateMatch mirrors the filer's strong-comparison canonicalization:
|
||||
// quotes are stripped on both sides and a weak (W/) candidate never matches.
|
||||
func etagCandidateMatch(candidates []string, stored string) bool {
|
||||
@@ -955,7 +985,7 @@ func TestPutToFilerRoutedAmbiguousEmptyPutAbsentNotRecreated(t *testing.T) {
|
||||
txnErr: status.Error(codes.Unavailable, "response lost after commit"),
|
||||
}
|
||||
// The committed empty object is deleted before recovery looks it up.
|
||||
filerImpl.beforeLookup = func() {
|
||||
filerImpl.beforeLookup = func(string) {
|
||||
delete(filerImpl.entries, "/buckets/b/o")
|
||||
}
|
||||
filerAddr := startFakeFiler(t, filerImpl)
|
||||
@@ -1279,3 +1309,12 @@ func TestSuspendedPutIfMatchRecheckedWhenAltKeyIgnored(t *testing.T) {
|
||||
t.Fatalf("condition keys = %v, want [\"\" \"/buckets/b/k.versions/v_v1\"]", conditionKeys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTxnConditionHoldsQuotedStoredETag(t *testing.T) {
|
||||
stored := &filer_pb.Entry{Extended: map[string][]byte{s3_constants.ExtETagKey: []byte(`"abc123"`)}}
|
||||
match := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{Kind: filer_pb.WriteCondition_IF_ETAG_MATCH, Etags: []string{`"abc123"`}}}}
|
||||
notMatch := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{Kind: filer_pb.WriteCondition_IF_ETAG_NOT_MATCH, Etags: []string{"abc123"}}}}
|
||||
if !txnConditionHolds(match, stored) || txnConditionHolds(notMatch, stored) {
|
||||
t.Fatalf("a quoted stored ETag must compare equal to the same ETag")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user