diff --git a/weed/s3api/s3api_internal_lifecycle.go b/weed/s3api/s3api_internal_lifecycle.go index 80958636e9..2c89b3bf4b 100644 --- a/weed/s3api/s3api_internal_lifecycle.go +++ b/weed/s3api/s3api_internal_lifecycle.go @@ -15,6 +15,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" stats_collect "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/util" ) // LifecycleDelete executes one (rule, action) verdict: re-fetch, identity @@ -104,39 +105,24 @@ func (s3a *S3ApiServer) lifecycleDispatch(ctx context.Context, req *s3_lifecycle } return done(), nil case s3_constants.VersioningSuspended: - // Best-effort null delete; NotFound is benign. Route it when an - // owner resolves — on the default filer an absent entry deletes as - // success, and a null version that only reached the owner would - // linger. Lifecycle never bypasses retention. - nullDeleted := false - if owner := s3a.routableWriteOwner(req.Bucket, req.ObjectPath); owner != "" { - worm, lockErr := s3a.isObjectLockEnabled(req.Bucket) - if lockErr == nil { - code, settled := s3a.routedDeleteNullVersion(owner, req.Bucket, req.ObjectPath, worm, false, metadataOnly, nil) - if settled { - if code == s3err.ErrAccessDenied { - return retryLater("ACCESS_DENIED: deleteNullVersion: object lock"), nil - } - if code != s3err.ErrNone { - return retryLater("TRANSPORT_ERROR: routedDeleteNullVersion: " + s3err.GetAPIError(code).Code), nil - } - nullDeleted = true - } - } - } - if !nullDeleted { - if err := s3a.deleteSpecificObjectVersion(ctx, req.Bucket, req.ObjectPath, "null", metadataOnly); err != nil { - if !errors.Is(err, filer_pb.ErrNotFound) && !errors.Is(err, ErrVersionNotFound) { - return retryLater("TRANSPORT_ERROR: deleteNullVersion: " + err.Error()), nil - } - } - } - // The null delete is done, so the marker must follow even past - // the deadline: without it a retry finds an older version latest. - if _, resp := s3a.lifecycleDeleteMarker(req.Bucket, req.ObjectPath, ""); resp != nil { + // Marker first: the pointer outranks the null object, so the key + // reads as deleted once it lands, and a null delete that fails + // after it leaves a noncurrent version instead of exposing an + // older one to a retry that then finds no null to expire. + nullEntry, resp := s3a.lifecycleNullVersion(req, entry) + if resp != nil { return resp, nil } - return done(), nil + if err := ctx.Err(); err != nil { + return nil, err + } + if _, resp := s3a.lifecycleDeleteMarker(req.Bucket, req.ObjectPath, lifecycleAddressedVersion(req, entry)); resp != nil { + return resp, nil + } + if nullEntry == nil { + return done(), nil + } + return s3a.lifecycleDeleteNullVersion(req, nullEntry, metadataOnly), nil default: err := s3a.WithFilerClient(false, func(c filer_pb.SeaweedFilerClient) error { return s3a.deleteUnversionedObjectWithClient(ctx, c, req.Bucket, req.ObjectPath, metadataOnly) @@ -291,6 +277,77 @@ func (s3a *S3ApiServer) lifecycleDeleteMarkerIf(bucket, object, normalized, expe return false, retryLater("LATEST_CHANGED: createDeleteMarker") } +// lifecycleNullVersion reads the null object the suspended expiration will +// delete. When it is the version validated for expiration, a different or +// missing entry resolves the request as stale. +func (s3a *S3ApiServer) lifecycleNullVersion(req *s3_lifecycle_pb.LifecycleDeleteRequest, validated *filer_pb.Entry) (*filer_pb.Entry, *s3_lifecycle_pb.LifecycleDeleteResponse) { + nullEntry, err := s3a.getObjectEntryRoutedByKey(req.Bucket, req.ObjectPath) + if err != nil && !isObjectEntryMiss(err) { + return nil, retryLater("TRANSPORT_ERROR: null version lookup: " + err.Error()) + } + if err != nil || !isNullVersionObject(nullEntry, req.ObjectPath) { + nullEntry = nil + } + if lifecycleAddressedVersion(req, validated) == "null" && (nullEntry == nil || !identityMatches(computeEntryIdentity(nullEntry), computeEntryIdentity(validated))) { + return nil, noopResolved("STALE_IDENTITY") + } + return nullEntry, nil +} + +// lifecycleDeleteNullVersion deletes the null version once the marker hides +// it, even past the attempt deadline. Only the entry read before the marker is +// deleted: a null object written since is newer and stays. Lifecycle never +// bypasses retention. +func (s3a *S3ApiServer) lifecycleDeleteNullVersion(req *s3_lifecycle_pb.LifecycleDeleteRequest, nullEntry *filer_pb.Entry, metadataOnly bool) *s3_lifecycle_pb.LifecycleDeleteResponse { + if owner := s3a.routableWriteOwner(req.Bucket, req.ObjectPath); owner != "" { + worm, lockErr := s3a.isObjectLockEnabled(req.Bucket) + if lockErr != nil { + return retryLater("TRANSPORT_ERROR: object lock lookup: " + lockErr.Error()) + } + guard := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{ + Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL, + ExpectedEntry: nullEntry, + }}} + switch code, _ := s3a.routedDeleteNullVersion(owner, req.Bucket, req.ObjectPath, worm, false, metadataOnly, guard); code { + case s3err.ErrNone, s3err.ErrPreconditionFailed: + return done() + case s3err.ErrAccessDenied: + return retryLater("ACCESS_DENIED: deleteNullVersion: object lock") + default: + return retryLater("TRANSPORT_ERROR: routedDeleteNullVersion: " + s3err.GetAPIError(code).Code) + } + } + // Off the ring, writers serialize on the object write lock instead. + dir, name := util.NewFullPath(s3a.bucketDir(req.Bucket), req.ObjectPath).DirAndName() + var opErr error + code := s3a.withObjectWriteLock(req.Bucket, req.ObjectPath, func() s3err.ErrorCode { + current, err := s3a.getEntry(dir, name) + switch { + case errors.Is(err, filer_pb.ErrNotFound): + return s3err.ErrPreconditionFailed + case err != nil: + opErr = err + return s3err.ErrInternalError + case !identityMatches(computeEntryIdentity(current), computeEntryIdentity(nullEntry)): + return s3err.ErrPreconditionFailed + } + return s3err.ErrNone + }, func() s3err.ErrorCode { + if opErr = s3a.deleteSpecificObjectVersion(context.Background(), req.Bucket, req.ObjectPath, "null", metadataOnly); opErr != nil { + return s3err.ErrInternalError + } + return s3err.ErrNone + }) + switch { + case code == s3err.ErrNone || code == s3err.ErrPreconditionFailed: + return done() + case opErr != nil: + return retryLater("TRANSPORT_ERROR: deleteNullVersion: " + opErr.Error()) + default: + return retryLater("TRANSPORT_ERROR: deleteNullVersion: " + s3err.GetAPIError(code).Code) + } +} + // pointerLatestVersion reads the latest version off a .versions pointer the // way getLatestObjectVersion does: a named pointer with its file outranks the // null object, the null-is-latest signal names the null version, and anything diff --git a/weed/s3api/s3api_internal_lifecycle_test.go b/weed/s3api/s3api_internal_lifecycle_test.go index 02b3d2768f..6ef12b033f 100644 --- a/weed/s3api/s3api_internal_lifecycle_test.go +++ b/weed/s3api/s3api_internal_lifecycle_test.go @@ -3,6 +3,8 @@ package s3api import ( "bytes" "context" + "errors" + "fmt" "net" "strconv" "testing" @@ -540,7 +542,9 @@ func TestLifecycleDelete_ExpiredAttemptWritesNoMarker(t *testing.T) { } } -func TestLifecycleDelete_DeadlineDuringNullDeleteStillWritesMarker(t *testing.T) { +// suspendedNullServer serves a suspended bucket whose latest is the null +// object, with an older version that a lost delete marker would expose. +func suspendedNullServer(t *testing.T, routed bool) (*S3ApiServer, *ambiguousPutFiler, *filer_pb.Entry) { olderId := generateVersionId(true) older := &filer_pb.Entry{ Name: "v_" + olderId, @@ -548,38 +552,290 @@ func TestLifecycleDelete_DeadlineDuringNullDeleteStillWritesMarker(t *testing.T) Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 3}}, Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte(olderId)}, } - latest := &filer_pb.Entry{ + null := &filer_pb.Entry{ Name: "obj", Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3}, Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}}, Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("null")}, } fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{ - "/buckets/b/obj": proto.Clone(latest).(*filer_pb.Entry), + "/buckets/b/obj": proto.Clone(null).(*filer_pb.Entry), "/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}}, "/buckets/b/obj.versions/" + older.Name: proto.Clone(older).(*filer_pb.Entry), }} - ctx, cancel := context.WithCancel(s3LocalPeerCtx()) - defer cancel() - fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { cancel() } addr := startFakeFiler(t, fake) s3a := newPutTestServer(t, addr) s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningSuspended}) - s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr) - s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1) + if routed { + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1) + } + return s3a, fake, null +} + +func isMarkerTxn(req *filer_pb.ObjectTransactionRequest) bool { + for _, m := range req.Mutations { + if m.Type == filer_pb.ObjectMutation_PUT { + return true + } + } + return false +} + +func requireLatestIsMarker(t *testing.T, s3a *S3ApiServer) { + t.Helper() + got, err := s3a.getLatestObjectVersion("b", "obj") + require.NoError(t, err) + require.Equal(t, "true", string(got.Extended[s3_constants.ExtDeleteMarkerKey]), "the older version must stay hidden behind the marker") +} + +func TestLifecycleDelete_DeadlineDuringMarkerStillDeletesNull(t *testing.T) { + s3a, fake, null := suspendedNullServer(t, true) + ctx, cancel := context.WithCancel(s3LocalPeerCtx()) + defer cancel() + fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { cancel() } resp, err := s3a.LifecycleDelete(ctx, &s3_lifecycle_pb.LifecycleDeleteRequest{ Bucket: "b", ObjectPath: "obj", ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, - ExpectedIdentity: computeEntryIdentity(latest), + ExpectedIdentity: computeEntryIdentity(null), }) require.NoError(t, err) require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason) - require.Equal(t, 2, fake.txnCount, "the null delete and the marker") - got, err := s3a.getLatestObjectVersion("b", "obj") + require.Equal(t, 2, fake.txnCount, "the marker and the null delete") + require.NotContains(t, fake.entries, "/buckets/b/obj") + requireLatestIsMarker(t, s3a) +} + +func TestLifecycleDelete_DeadlineDuringNullReadWritesNoMarker(t *testing.T) { + nullReads := func(cancelAt int, cancel func()) int { + s3a, fake, null := suspendedNullServer(t, true) + reads := 0 + fake.beforeLookup = func(key string) { + if key != "/buckets/b/obj" || fake.txnCount > 0 { + return + } + if reads++; reads == cancelAt { + cancel() + } + } + ctx, stop := context.WithCancel(s3LocalPeerCtx()) + defer stop() + if cancel == nil { + cancel = stop + } + _, err := s3a.LifecycleDelete(ctx, &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, + ExpectedIdentity: computeEntryIdentity(null), + }) + if cancelAt > 0 { + require.ErrorIs(t, err, context.Canceled) + require.Zero(t, fake.txnCount, "an attempt past its deadline must not write a delete marker") + require.Contains(t, fake.entries, "/buckets/b/obj") + } + return reads + } + last := nullReads(0, nil) + require.Positive(t, last) + nullReads(last, nil) +} + +// A marker write that fails leaves the null version in place, so the retry +// still finds it latest and finishes the expiration. +func TestLifecycleDelete_SuspendedMarkerFailureLeavesNullForRetry(t *testing.T) { + for _, routed := range []bool{true, false} { + t.Run(fmt.Sprintf("routed=%v", routed), func(t *testing.T) { + s3a, fake, null := suspendedNullServer(t, routed) + failMarker := true + fake.txnCheck = func(req *filer_pb.ObjectTransactionRequest) { + fake.txnNoApply = failMarker && isMarkerTxn(req) + fake.txnErr = nil + if fake.txnNoApply { + fake.txnErr = errors.New("owner unavailable") + } + } + fake.createErr = errors.New("filer unavailable") + req := &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, + ExpectedIdentity: computeEntryIdentity(null), + } + resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), req) + require.NoError(t, err) + require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_RETRY_LATER, resp.Outcome, resp.Reason) + require.Contains(t, fake.entries, "/buckets/b/obj", "the null version must outlive a failed marker") + + failMarker, fake.createErr = false, nil + resp, err = s3a.LifecycleDelete(s3LocalPeerCtx(), req) + require.NoError(t, err) + require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason) + require.NotContains(t, fake.entries, "/buckets/b/obj") + requireLatestIsMarker(t, s3a) + }) + } +} + +// A routed null delete that fails retries rather than deleting through the +// default filer, which may not have the owner's newer null object yet. +func TestLifecycleDelete_RoutedNullDeleteFailureRetries(t *testing.T) { + s3a, fake, null := suspendedNullServer(t, true) + fake.txnCheck = func(req *filer_pb.ObjectTransactionRequest) { + fake.txnNoApply = !isMarkerTxn(req) + fake.txnErr = nil + if fake.txnNoApply { + fake.txnErr = errors.New("owner unavailable") + } + } + resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, + ExpectedIdentity: computeEntryIdentity(null), + }) require.NoError(t, err) - require.Equal(t, "true", string(got.Extended[s3_constants.ExtDeleteMarkerKey]), "the older version must stay hidden behind the marker") + require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_RETRY_LATER, resp.Outcome, resp.Reason) + require.Contains(t, fake.entries, "/buckets/b/obj") + requireLatestIsMarker(t, s3a) +} + +func TestLifecycleDelete_RoutedNullDeleteObjectLockLookupFailureRetries(t *testing.T) { + s3a, fake, null := suspendedNullServer(t, true) + fake.txnCheck = func(req *filer_pb.ObjectTransactionRequest) { + if isMarkerTxn(req) { + s3a.bucketConfigCache.Remove("b") + fake.lookupErr, fake.lookupFailKey = status.Error(codes.Unavailable, "filer unavailable"), "/buckets/b" + } + } + resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, + ExpectedIdentity: computeEntryIdentity(null), + }) + require.NoError(t, err) + require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_RETRY_LATER, resp.Outcome, resp.Reason) + require.Equal(t, 1, fake.txnCount, "only the marker") + require.Contains(t, fake.entries, "/buckets/b/obj") +} + +// A null object written after the expiration read the old one is not deleted. +func TestLifecycleDelete_SuspendedNullRewrittenBeforeDeleteSurvives(t *testing.T) { + for _, routed := range []bool{true, false} { + t.Run(fmt.Sprintf("routed=%v", routed), func(t *testing.T) { + s3a, fake, null := suspendedNullServer(t, routed) + rewritten := proto.Clone(null).(*filer_pb.Entry) + rewritten.Attributes.Mtime++ + rewritten.Chunks = []*filer_pb.FileChunk{{FileId: "3,03637037d6", Size: 3}} + rewrite := func() { fake.entries["/buckets/b/obj"] = proto.Clone(rewritten).(*filer_pb.Entry) } + fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { + if fake.txnCount == 1 { + rewrite() + } + } + fake.beforeCreate = rewrite + var locked []string + s3a.newObjectWriteLock = func(bucket, object string) (objectWriteLock, error) { + locked = append(locked, bucket+"/"+object) + return noopObjectWriteLock{}, nil + } + resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, + ExpectedIdentity: computeEntryIdentity(null), + }) + require.NoError(t, err) + require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason) + require.Contains(t, fake.entries, "/buckets/b/obj") + require.Equal(t, "3,03637037d6", fake.entries["/buckets/b/obj"].Chunks[0].GetFileIdString()) + if !routed { + require.Equal(t, []string{"b/obj", "b/obj"}, locked, "the marker, then the recheck and the delete, must hold the object write lock") + } + }) + } +} + +// A null PUT that lands after a suspended expiration validated a named version +// is neither hidden nor deleted, whether it lands before the null read or +// before the routed marker commit. +func TestLifecycleDelete_SuspendedPutOverNamedVersionSurvives(t *testing.T) { + old := &filer_pb.Entry{ + Name: "v_old", + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}}, + Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")}, + } + fresh := &filer_pb.Entry{ + Name: "obj", + Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5}, + Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}}, + Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("null")}, + } + cases := []struct { + name string + routed bool + atRead bool + }{ + {"routed/at-null-read", true, true}, + {"routed/at-marker", true, false}, + {"unrouted/at-null-read", false, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{ + "/buckets/b/obj.versions/v_old": proto.Clone(old).(*filer_pb.Entry), + "/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{ + s3_constants.ExtLatestVersionIdKey: []byte("old"), + s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"), + }}, + }} + put := func() { + if _, ok := fake.entries["/buckets/b/obj"]; ok { + return + } + fake.entries["/buckets/b/obj"] = proto.Clone(fresh).(*filer_pb.Entry) + pointer := fake.entries["/buckets/b/obj.versions"] + delete(pointer.Extended, s3_constants.ExtLatestVersionIdKey) + delete(pointer.Extended, s3_constants.ExtLatestVersionFileNameKey) + pointer.Extended[s3_constants.ExtNullVersionIsLatestKey] = []byte("true") + } + if tc.atRead { + fake.beforeLookup = func(key string) { + if key == "/buckets/b/obj" { + put() + } + } + } else { + fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { put() } + } + addr := startFakeFiler(t, fake) + s3a := newPutTestServer(t, addr) + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningSuspended}) + if tc.routed { + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1) + } + resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{ + Bucket: "b", + ObjectPath: "obj", + VersionId: "old", + ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, + ExpectedIdentity: computeEntryIdentity(old), + }) + require.NoError(t, err) + require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED, resp.Outcome, resp.Reason) + require.Equal(t, "VERSION_NOT_LATEST", resp.Reason) + require.Contains(t, fake.entries, "/buckets/b/obj") + for key, e := range fake.entries { + require.NotEqual(t, "true", string(e.Extended[s3_constants.ExtDeleteMarkerKey]), key) + } + }) + } } // A timed-out attempt can still be running when its retry writes the marker; @@ -605,7 +861,7 @@ func TestLifecycleDelete_OverlappingAttemptWritesOneMarker(t *testing.T) { outcome s3_lifecycle_pb.LifecycleDeleteOutcome }{ {s3_constants.VersioningEnabled, 1, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED}, - {s3_constants.VersioningSuspended, 2, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}, + {s3_constants.VersioningSuspended, 1, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}, } for _, tc := range cases { t.Run(tc.versioning, func(t *testing.T) { diff --git a/weed/s3api/s3api_object_handlers_put_ambiguous_test.go b/weed/s3api/s3api_object_handlers_put_ambiguous_test.go index 9b8a4850b5..8e830e5af1 100644 --- a/weed/s3api/s3api_object_handlers_put_ambiguous_test.go +++ b/weed/s3api/s3api_object_handlers_put_ambiguous_test.go @@ -155,7 +155,7 @@ type ambiguousPutFiler struct { // beforeLookup runs inside LookupDirectoryEntry, letting a test delete a // just-committed entry before recovery reads it. beforeCreate func() - beforeLookup func() + beforeLookup func(key string) beforeAssign func() stores int conds []*filer_pb.WriteCondition @@ -232,11 +232,18 @@ func putConditionSatisfied(cond *filer_pb.WriteCondition, current *filer_pb.Entr return true } +func (f *ambiguousPutFiler) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.entries, req.Directory+"/"+req.Name) + return &filer_pb.DeleteEntryResponse{}, nil +} + func (f *ambiguousPutFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) { f.mu.Lock() defer f.mu.Unlock() if f.beforeLookup != nil { - f.beforeLookup() + f.beforeLookup(req.Directory + "/" + req.Name) } if f.lookupErr != nil && req.Directory+"/"+req.Name == f.lookupFailKey { return nil, f.lookupErr @@ -279,30 +286,23 @@ func (f *ambiguousPutFiler) ObjectTransaction(_ context.Context, req *filer_pb.O } } } - for _, clause := range req.GetCondition().GetClauses() { - var md5Hex string - if current != nil && current.Attributes != nil { - md5Hex = fmt.Sprintf("%x", current.Attributes.Md5) - } - ok := true - switch clause.Kind { - case filer_pb.WriteCondition_IF_NOT_EXISTS: - ok = current == nil - case filer_pb.WriteCondition_IF_EXISTS: - ok = current != nil - case filer_pb.WriteCondition_IF_ETAG_MATCH: - ok = current != nil && etagCandidateMatch(clause.Etags, md5Hex) - case filer_pb.WriteCondition_IF_ETAG_NOT_MATCH: - ok = current == nil || !etagCandidateMatch(clause.Etags, md5Hex) - case filer_pb.WriteCondition_IF_ENTRY_EQUAL: - ok = (current == nil) == (clause.ExpectedEntry == nil) && (current == nil || proto.Equal(current, clause.ExpectedEntry)) - } - if !ok { - return &filer_pb.ObjectTransactionResponse{ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil - } + if !txnConditionHolds(req.GetCondition(), current) { + return &filer_pb.ObjectTransactionResponse{ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil } if !f.txnNoApply { - for _, m := range req.Mutations { + for i, m := range req.Mutations { + if m.Condition != nil { + target := m.ConditionKey + if target == "" && m.Type == filer_pb.ObjectMutation_PUT { + target = m.Directory + "/" + m.Entry.Name + } else if target == "" { + target = m.Directory + "/" + m.Name + } + if !txnConditionHolds(m.Condition, f.entries[target]) { + failed := int32(i) + return &filer_pb.ObjectTransactionResponse{ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED, FailedMutation: &failed}, nil + } + } switch m.Type { case filer_pb.ObjectMutation_PUT: entry := proto.Clone(m.Entry).(*filer_pb.Entry) @@ -323,6 +323,36 @@ func (f *ambiguousPutFiler) ObjectTransaction(_ context.Context, req *filer_pb.O return &filer_pb.ObjectTransactionResponse{}, f.txnErr } +// txnConditionHolds evaluates an ObjectTransaction condition, request-level or +// on one mutation, the way the filer does for the kinds the tests send. +func txnConditionHolds(cond *filer_pb.WriteCondition, current *filer_pb.Entry) bool { + for _, clause := range cond.GetClauses() { + var etag string + if v := current.GetExtended()[s3_constants.ExtETagKey]; len(v) > 0 { + etag = strings.Trim(string(v), `"`) + } else if current != nil && current.Attributes != nil { + etag = fmt.Sprintf("%x", current.Attributes.Md5) + } + ok := true + switch clause.Kind { + case filer_pb.WriteCondition_IF_NOT_EXISTS: + ok = current == nil + case filer_pb.WriteCondition_IF_EXISTS: + ok = current != nil + case filer_pb.WriteCondition_IF_ETAG_MATCH: + ok = current != nil && etagCandidateMatch(clause.Etags, etag) + case filer_pb.WriteCondition_IF_ETAG_NOT_MATCH: + ok = current == nil || !etagCandidateMatch(clause.Etags, etag) + case filer_pb.WriteCondition_IF_ENTRY_EQUAL: + ok = (current == nil) == (clause.ExpectedEntry == nil) && (current == nil || proto.Equal(current, clause.ExpectedEntry)) + } + if !ok { + return false + } + } + return true +} + // etagCandidateMatch mirrors the filer's strong-comparison canonicalization: // quotes are stripped on both sides and a weak (W/) candidate never matches. func etagCandidateMatch(candidates []string, stored string) bool { @@ -955,7 +985,7 @@ func TestPutToFilerRoutedAmbiguousEmptyPutAbsentNotRecreated(t *testing.T) { txnErr: status.Error(codes.Unavailable, "response lost after commit"), } // The committed empty object is deleted before recovery looks it up. - filerImpl.beforeLookup = func() { + filerImpl.beforeLookup = func(string) { delete(filerImpl.entries, "/buckets/b/o") } filerAddr := startFakeFiler(t, filerImpl) @@ -1279,3 +1309,12 @@ func TestSuspendedPutIfMatchRecheckedWhenAltKeyIgnored(t *testing.T) { t.Fatalf("condition keys = %v, want [\"\" \"/buckets/b/k.versions/v_v1\"]", conditionKeys) } } + +func TestTxnConditionHoldsQuotedStoredETag(t *testing.T) { + stored := &filer_pb.Entry{Extended: map[string][]byte{s3_constants.ExtETagKey: []byte(`"abc123"`)}} + match := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{Kind: filer_pb.WriteCondition_IF_ETAG_MATCH, Etags: []string{`"abc123"`}}}} + notMatch := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{Kind: filer_pb.WriteCondition_IF_ETAG_NOT_MATCH, Etags: []string{"abc123"}}}} + if !txnConditionHolds(match, stored) || txnConditionHolds(notMatch, stored) { + t.Fatalf("a quoted stored ETag must compare equal to the same ETag") + } +}