mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-08 15:41:15 +02:00
s3: make RenameObject idempotent for a retried request (#11178)
* feat(s3): make RenameObject idempotent for a retried request - #10661 A rename that succeeds but whose response is lost leaves the client with no safe move: retrying returned 404, because the source is already gone, so a retry was indistinguishable from a rename that never happened. The destination now carries what the rename that created it was, under x-seaweedfs-rename-token: the client's token, the source key and the time. A retry that names the same token and the same source and destination is answered 200 without touching anything. The same token sent for a different rename is refused with 409 rather than silently answered, and a token older than 24 hours is treated as unrelated so a key cannot answer for a request indefinitely. Requests without the header behave exactly as before. * s3: answer a reused rename token with 409, not 400 The PR promised Conflict and the code returned Bad Request. 400 tells a client its request was malformed and invites it to give up; this request is well formed and resending it unchanged will not help, because what it collides with is a rename the same token already stands for. The status code is now asserted in a test, since it is the part of this behaviour a client actually acts on. * Update weed/s3api/s3err/s3api_errors.go Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3: fix rename token review notes - ErrIdempotentParameterMismatch returns 409 Conflict, not 400. The comment and TestRenameTokenReuseAnswersConflict both expect 409; the code regressed to 400 in a later commit. - stampRenameToken: clarify that markRenameToken mutates srcEntry in place, so the token reaches the destination via the move regardless of whether the UpdateEntry succeeds. The precondition only guards the pre-move write, not the move itself. - Extract the handler retry branch into retryRenameDecision and add TestRetryRenameDecision, covering the source-still-exists fallthrough that was previously reasoned about but not tested. * s3: IdempotentParameterMismatch returns 400, matching AWS docs The AWS S3 RenameObject API documentation specifies HTTP Status Code: 400 for IdempotencyParameterMismatch. Revert the previous 409 change and align the comment and test with the documented behavior. --------- Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Chris Lu <chris.lu@gmail.com>
This commit is contained in:
co-authored by
Chris Lu
devin-ai-integration[bot]
Chris Lu
parent
3f9b05946b
commit
f99c4a1f14
@@ -107,6 +107,9 @@ const (
|
|||||||
AmzCopySourceIfUnmodifiedSince = "X-Amz-Copy-Source-If-Unmodified-Since"
|
AmzCopySourceIfUnmodifiedSince = "X-Amz-Copy-Source-If-Unmodified-Since"
|
||||||
|
|
||||||
// RenameObject
|
// RenameObject
|
||||||
|
// AmzClientToken makes a rename idempotent. The AWS SDKs fill it in on every
|
||||||
|
// call, so it arrives on requests that were never written with it in mind.
|
||||||
|
AmzClientToken = "X-Amz-Client-Token"
|
||||||
AmzRenameSource = "X-Amz-Rename-Source"
|
AmzRenameSource = "X-Amz-Rename-Source"
|
||||||
AmzRenameSourceIfMatch = "X-Amz-Rename-Source-If-Match"
|
AmzRenameSourceIfMatch = "X-Amz-Rename-Source-If-Match"
|
||||||
AmzRenameSourceIfNoneMatch = "X-Amz-Rename-Source-If-None-Match"
|
AmzRenameSourceIfNoneMatch = "X-Amz-Rename-Source-If-None-Match"
|
||||||
@@ -157,6 +160,12 @@ const (
|
|||||||
SeaweedFSSSEKMSEncryptionContext = "x-seaweedfs-sse-kms-encryption-context" // Encryption context for multipart upload SSE-KMS inheritance
|
SeaweedFSSSEKMSEncryptionContext = "x-seaweedfs-sse-kms-encryption-context" // Encryption context for multipart upload SSE-KMS inheritance
|
||||||
SeaweedFSSSEKMSBaseIV = "x-seaweedfs-sse-kms-base-iv" // Base IV for multipart upload SSE-KMS (for IV offset calculation)
|
SeaweedFSSSEKMSBaseIV = "x-seaweedfs-sse-kms-base-iv" // Base IV for multipart upload SSE-KMS (for IV offset calculation)
|
||||||
|
|
||||||
|
// SeaweedFSRenameToken records the x-amz-client-token of the rename that put an
|
||||||
|
// object at its key, together with the source that rename named. It rides on the
|
||||||
|
// object itself, so every gateway reads the same answer for a retry of that
|
||||||
|
// rename, and it is dropped whenever the key is written again.
|
||||||
|
SeaweedFSRenameToken = "x-seaweedfs-rename-token"
|
||||||
|
|
||||||
// Multipart upload metadata keys for SSE-S3
|
// Multipart upload metadata keys for SSE-S3
|
||||||
SeaweedFSSSES3Encryption = "x-seaweedfs-sse-s3-encryption" // Encryption type for multipart upload SSE-S3 inheritance
|
SeaweedFSSSES3Encryption = "x-seaweedfs-sse-s3-encryption" // Encryption type for multipart upload SSE-S3 inheritance
|
||||||
SeaweedFSSSES3BaseIV = "x-seaweedfs-sse-s3-base-iv" // Base IV for multipart upload SSE-S3 (for IV offset calculation)
|
SeaweedFSSSES3BaseIV = "x-seaweedfs-sse-s3-base-iv" // Base IV for multipart upload SSE-S3 (for IV offset calculation)
|
||||||
|
|||||||
@@ -2,10 +2,12 @@ package s3api
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||||
@@ -31,6 +33,8 @@ var renameSourceConditionalHeaders = sourceConditionalHeaderNames{
|
|||||||
//
|
//
|
||||||
// The object is moved by the filer's AtomicRenameEntry, so its bytes are never
|
// The object is moved by the filer's AtomicRenameEntry, so its bytes are never
|
||||||
// read or rewritten and its metadata (ETag, tags, SSE keys) travels unchanged.
|
// read or rewritten and its metadata (ETag, tags, SSE keys) travels unchanged.
|
||||||
|
// x-amz-client-token travels with it too, so a rename whose response was lost
|
||||||
|
// answers its own retry instead of the NoSuchKey its vanished source would give.
|
||||||
// Versioned buckets are rejected: the move would have to rebuild the .versions
|
// Versioned buckets are rejected: the move would have to rebuild the .versions
|
||||||
// chain, and AWS itself only offers RenameObject on directory buckets, which
|
// chain, and AWS itself only offers RenameObject on directory buckets, which
|
||||||
// cannot be versioned.
|
// cannot be versioned.
|
||||||
@@ -92,20 +96,39 @@ func (s3a *S3ApiServer) RenameObjectHandler(w http.ResponseWriter, r *http.Reque
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
clientToken := r.Header.Get(s3_constants.AmzClientToken)
|
||||||
|
renameSource := r.Header.Get(s3_constants.AmzRenameSource)
|
||||||
|
|
||||||
errCode = s3a.withRenameWriteLocks(bucket, srcObject, dstObject, func() s3err.ErrorCode {
|
errCode = s3a.withRenameWriteLocks(bucket, srcObject, dstObject, func() s3err.ErrorCode {
|
||||||
entry, err := s3a.resolveCopySourceEntry(bucket, srcObject, "", "")
|
entry, err := s3a.resolveCopySourceEntry(bucket, srcObject, "", "")
|
||||||
srcIsPrefixObject := entry.IsPrefixObject()
|
srcIsPrefixObject := entry.IsPrefixObject()
|
||||||
entry = prefixObjectSource(entry)
|
entry = prefixObjectSource(entry)
|
||||||
if errCode := classifyCopySourceError(entry, err); errCode != s3err.ErrNone {
|
srcErrCode := classifyCopySourceError(entry, err)
|
||||||
|
|
||||||
|
dstEntry, errCode := s3a.lookupRenameDestination(bucket, dstObject)
|
||||||
|
if errCode != s3err.ErrNone {
|
||||||
return errCode
|
return errCode
|
||||||
}
|
}
|
||||||
|
if errCode, settled := retryRenameDecision(dstEntry, clientToken, renameSource, dstObject, srcErrCode); settled {
|
||||||
|
return errCode
|
||||||
|
}
|
||||||
|
if srcErrCode != s3err.ErrNone {
|
||||||
|
return srcErrCode
|
||||||
|
}
|
||||||
if errCode := validateSourceConditionalHeaders(r, entry, renameSourceConditionalHeaders); errCode != s3err.ErrNone {
|
if errCode := validateSourceConditionalHeaders(r, entry, renameSourceConditionalHeaders); errCode != s3err.ErrNone {
|
||||||
return errCode
|
return errCode
|
||||||
}
|
}
|
||||||
if errCode := s3a.checkConditionalHeaders(r, bucket, dstObject); errCode != s3err.ErrNone {
|
if errCode := s3a.checkConditionalHeaders(r, bucket, dstObject); errCode != s3err.ErrNone {
|
||||||
return errCode
|
return errCode
|
||||||
}
|
}
|
||||||
return s3a.renameObjectEntry(r.Context(), bucket, srcObject, dstObject, entry, srcIsPrefixObject)
|
|
||||||
|
// AtomicRenameEntry moves a directory by moving everything under it, and the keys
|
||||||
|
// nested under either end of this rename are not part of what is being renamed.
|
||||||
|
keyHoldsNestedKeys := srcIsPrefixObject || (dstEntry != nil && dstEntry.IsDirectory)
|
||||||
|
if clientToken != "" {
|
||||||
|
s3a.stampRenameToken(bucket, srcObject, dstObject, entry, clientToken, renameSource, keyHoldsNestedKeys)
|
||||||
|
}
|
||||||
|
return s3a.renameObjectEntry(r.Context(), bucket, srcObject, dstObject, entry, keyHoldsNestedKeys)
|
||||||
})
|
})
|
||||||
if errCode != s3err.ErrNone {
|
if errCode != s3err.ErrNone {
|
||||||
s3err.WriteErrorResponse(w, r, errCode)
|
s3err.WriteErrorResponse(w, r, errCode)
|
||||||
@@ -207,6 +230,139 @@ func (s3a *S3ApiServer) authorizeRenameSource(r *http.Request, bucket, srcObject
|
|||||||
return s3a.iam.AuthorizeObjectDelete(r, identity, bucket, srcObject, "")
|
return s3a.iam.AuthorizeObjectDelete(r, identity, bucket, srcObject, "")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// renameTokenValidity bounds how long a committed rename answers for its own
|
||||||
|
// retry. An SDK gives up retrying long before this, so the window covers every
|
||||||
|
// retry there is, while a token that comes back days later no longer stands in
|
||||||
|
// for a move the caller means to happen now.
|
||||||
|
const renameTokenValidity = 24 * time.Hour
|
||||||
|
|
||||||
|
// renameToken is what a rename leaves on the object it moves, so that the same
|
||||||
|
// request replayed after its response was lost can be answered from the
|
||||||
|
// destination instead of from a source that is no longer there.
|
||||||
|
//
|
||||||
|
// Source is the x-amz-rename-source header as it was sent. A retry resends the
|
||||||
|
// request byte for byte, so equality on the raw value recognises it, and any
|
||||||
|
// other value is the token used for a different rename. Dest is the key the
|
||||||
|
// rename wrote, which tells a token that travelled here some other way - on a
|
||||||
|
// CopyObject of a renamed object, say - from one this rename left.
|
||||||
|
type renameToken struct {
|
||||||
|
Token string `json:"token"`
|
||||||
|
Source string `json:"source"`
|
||||||
|
Dest string `json:"dest"`
|
||||||
|
Unix int64 `json:"unix"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type renameTokenVerdict int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// renameTokenUnrelated: the destination carries no live token of this request's.
|
||||||
|
renameTokenUnrelated renameTokenVerdict = iota
|
||||||
|
// renameTokenSameRequest: this very rename already committed here.
|
||||||
|
renameTokenSameRequest
|
||||||
|
// renameTokenReused: the same token was sent for a different rename.
|
||||||
|
renameTokenReused
|
||||||
|
)
|
||||||
|
|
||||||
|
// classifyRenameToken reads what the destination object says about a request
|
||||||
|
// carrying clientToken.
|
||||||
|
func classifyRenameToken(dstEntry *filer_pb.Entry, clientToken, renameSource, dstObject string) renameTokenVerdict {
|
||||||
|
if clientToken == "" || dstEntry == nil {
|
||||||
|
return renameTokenUnrelated
|
||||||
|
}
|
||||||
|
stamped, found := dstEntry.Extended[s3_constants.SeaweedFSRenameToken]
|
||||||
|
if !found {
|
||||||
|
return renameTokenUnrelated
|
||||||
|
}
|
||||||
|
var token renameToken
|
||||||
|
if err := json.Unmarshal(stamped, &token); err != nil {
|
||||||
|
glog.Warningf("RenameObject: unreadable rename token on %s: %v", dstEntry.Name, err)
|
||||||
|
return renameTokenUnrelated
|
||||||
|
}
|
||||||
|
if token.Token != clientToken || token.Dest != dstObject || time.Since(time.Unix(token.Unix, 0)) > renameTokenValidity {
|
||||||
|
return renameTokenUnrelated
|
||||||
|
}
|
||||||
|
if token.Source != renameSource {
|
||||||
|
return renameTokenReused
|
||||||
|
}
|
||||||
|
return renameTokenSameRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
// retryRenameDecision says what to do with a request whose destination carries
|
||||||
|
// a rename token. settled=true means the request is answered (return errCode);
|
||||||
|
// settled=false means continue with the rename.
|
||||||
|
//
|
||||||
|
// A reused token is refused. A token from this very rename is answered as
|
||||||
|
// success only when the source is gone — the move already committed and only
|
||||||
|
// its response was lost. A source that is back is not that retry: the move is
|
||||||
|
// still to be made, and making it is what leaves the caller where it asked to
|
||||||
|
// be, so the decision falls through and the rename proceeds.
|
||||||
|
func retryRenameDecision(dstEntry *filer_pb.Entry, clientToken, renameSource, dstObject string, srcErrCode s3err.ErrorCode) (s3err.ErrorCode, bool) {
|
||||||
|
switch classifyRenameToken(dstEntry, clientToken, renameSource, dstObject) {
|
||||||
|
case renameTokenReused:
|
||||||
|
return s3err.ErrIdempotentParameterMismatch, true
|
||||||
|
case renameTokenSameRequest:
|
||||||
|
if srcErrCode == s3err.ErrNoSuchKey {
|
||||||
|
return s3err.ErrNone, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return s3err.ErrNone, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// markRenameToken puts the client token on the entry the rename moves.
|
||||||
|
func markRenameToken(srcEntry *filer_pb.Entry, clientToken, renameSource, dstObject string) error {
|
||||||
|
stamped, err := json.Marshal(renameToken{Token: clientToken, Source: renameSource, Dest: dstObject, Unix: time.Now().Unix()})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if srcEntry.Extended == nil {
|
||||||
|
srcEntry.Extended = make(map[string][]byte)
|
||||||
|
}
|
||||||
|
srcEntry.Extended[s3_constants.SeaweedFSRenameToken] = stamped
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// stampRenameToken records the client token on the object about to be moved, so
|
||||||
|
// that the move carries it to the destination.
|
||||||
|
//
|
||||||
|
// The token goes on before the move rather than after it: a move that commits
|
||||||
|
// and then loses its token is exactly the failure the token exists to cover,
|
||||||
|
// while a token left on a move that never happened simply rides along with the
|
||||||
|
// next attempt. Failing to record it costs this rename its idempotency and
|
||||||
|
// nothing else, so the move goes ahead either way.
|
||||||
|
func (s3a *S3ApiServer) stampRenameToken(bucket, srcObject, dstObject string, srcEntry *filer_pb.Entry, clientToken, renameSource string, keyHoldsNestedKeys bool) {
|
||||||
|
// The ETag as it stands now, read before the token joins it.
|
||||||
|
expected := map[string][]byte{s3_constants.ExtETagKey: srcEntry.Extended[s3_constants.ExtETagKey]}
|
||||||
|
|
||||||
|
if err := markRenameToken(srcEntry, clientToken, renameSource, dstObject); err != nil {
|
||||||
|
glog.Errorf("RenameObject %s: rename token for %s: %v", bucket, srcObject, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// renameKeyHoldingNestedKeys writes the destination out of this entry, so the
|
||||||
|
// token reaches it without a write of its own.
|
||||||
|
if keyHoldsNestedKeys {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// AtomicRenameEntry moves the entry as the filer holds it, so the token has to
|
||||||
|
// be on the source for the move to carry it. markRenameToken already mutated
|
||||||
|
// srcEntry.Extended in place, so the move carries the token to the destination
|
||||||
|
// whether or not this UpdateEntry succeeds — the write only persists the token
|
||||||
|
// on the source ahead of the move, and a lost write costs idempotency, not
|
||||||
|
// correctness. The precondition keeps the write off an object another gateway
|
||||||
|
// replaced in the meantime.
|
||||||
|
srcDir, _ := util.FullPath(s3a.toFilerPath(bucket, srcObject)).DirAndName()
|
||||||
|
if err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||||
|
return filer_pb.UpdateEntry(context.Background(), client, &filer_pb.UpdateEntryRequest{
|
||||||
|
Directory: srcDir,
|
||||||
|
Entry: srcEntry,
|
||||||
|
ExpectedExtended: expected,
|
||||||
|
})
|
||||||
|
}); err != nil {
|
||||||
|
glog.Warningf("RenameObject %s: record rename token on %s: %v", bucket, srcObject, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// withRenameWriteLocks holds the object write lock of both keys across the
|
// withRenameWriteLocks holds the object write lock of both keys across the
|
||||||
// precondition checks and the move. The keys are locked in a fixed order so a
|
// precondition checks and the move. The keys are locked in a fixed order so a
|
||||||
// rename in the opposite direction cannot deadlock against this one.
|
// rename in the opposite direction cannot deadlock against this one.
|
||||||
@@ -220,25 +376,32 @@ func (s3a *S3ApiServer) withRenameWriteLocks(bucket, srcObject, dstObject string
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s3a *S3ApiServer) renameObjectEntry(ctx context.Context, bucket, srcObject, dstObject string, srcEntry *filer_pb.Entry, srcIsPrefixObject bool) s3err.ErrorCode {
|
// lookupRenameDestination reads what the destination key already holds. A key
|
||||||
|
// nothing lives at is the ordinary case and comes back as a nil entry.
|
||||||
|
//
|
||||||
|
// The move overwrites an existing destination object. A directory there is not a
|
||||||
|
// conflict: it means other keys are nested under the destination key, and the
|
||||||
|
// object goes onto the directory they live in, the way a PutObject of that key
|
||||||
|
// would put it there.
|
||||||
|
func (s3a *S3ApiServer) lookupRenameDestination(bucket, dstObject string) (*filer_pb.Entry, s3err.ErrorCode) {
|
||||||
|
dstDir, dstName := util.FullPath(s3a.toFilerPath(bucket, dstObject)).DirAndName()
|
||||||
|
|
||||||
|
entry, err := s3a.getEntry(dstDir, dstName)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||||
|
return nil, s3err.ErrNone
|
||||||
|
}
|
||||||
|
glog.Errorf("RenameObject %s: destination %s: %v", bucket, dstObject, err)
|
||||||
|
return nil, s3err.ErrInternalError
|
||||||
|
}
|
||||||
|
return entry, s3err.ErrNone
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s3a *S3ApiServer) renameObjectEntry(ctx context.Context, bucket, srcObject, dstObject string, srcEntry *filer_pb.Entry, keyHoldsNestedKeys bool) s3err.ErrorCode {
|
||||||
srcDir, srcName := util.FullPath(s3a.toFilerPath(bucket, srcObject)).DirAndName()
|
srcDir, srcName := util.FullPath(s3a.toFilerPath(bucket, srcObject)).DirAndName()
|
||||||
dstDir, dstName := util.FullPath(s3a.toFilerPath(bucket, dstObject)).DirAndName()
|
dstDir, dstName := util.FullPath(s3a.toFilerPath(bucket, dstObject)).DirAndName()
|
||||||
|
|
||||||
// The move overwrites an existing destination object. A directory there is not a
|
if keyHoldsNestedKeys {
|
||||||
// conflict: it means other keys are nested under the destination key, and the
|
|
||||||
// object goes onto the directory they live in, the way a PutObject of that key
|
|
||||||
// would put it there.
|
|
||||||
dstHoldsNestedKeys := false
|
|
||||||
if existing, err := s3a.getEntry(dstDir, dstName); err == nil {
|
|
||||||
dstHoldsNestedKeys = existing.IsDirectory
|
|
||||||
} else if !errors.Is(err, filer_pb.ErrNotFound) {
|
|
||||||
glog.Errorf("RenameObject %s: destination %s: %v", bucket, dstObject, err)
|
|
||||||
return s3err.ErrInternalError
|
|
||||||
}
|
|
||||||
|
|
||||||
// AtomicRenameEntry moves a directory by moving everything under it, and the keys
|
|
||||||
// nested under either end of this rename are not part of what is being renamed.
|
|
||||||
if srcIsPrefixObject || dstHoldsNestedKeys {
|
|
||||||
return s3a.renameKeyHoldingNestedKeys(bucket, srcObject, dstObject, srcEntry)
|
return s3a.renameKeyHoldingNestedKeys(bucket, srcObject, dstObject, srcEntry)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package s3api
|
package s3api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"net/http"
|
"net/http"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -11,6 +12,7 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"google.golang.org/protobuf/proto"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestRenameSourceCandidates: AWS spells x-amz-rename-source as a bare key in
|
// TestRenameSourceCandidates: AWS spells x-amz-rename-source as a bare key in
|
||||||
@@ -144,6 +146,122 @@ func TestSourceConditionalHeaderPrecedence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRenameTokenAnswersItsOwnRetry walks the retry x-amz-client-token is there
|
||||||
|
// for: the rename commits, its response is lost, and the SDK resends the request
|
||||||
|
// unchanged. The move carries the token to the destination, so the destination
|
||||||
|
// can answer a retry that its own source no longer can.
|
||||||
|
func TestRenameTokenAnswersItsOwnRetry(t *testing.T) {
|
||||||
|
const clientToken = "rename-token-of-this-request"
|
||||||
|
const renameSource = "/bucket/src.txt"
|
||||||
|
|
||||||
|
entry := &filer_pb.Entry{
|
||||||
|
Name: "src.txt",
|
||||||
|
Extended: map[string][]byte{s3_constants.ExtETagKey: []byte("d41d8cd98f00b204e9800998ecf8427e")},
|
||||||
|
}
|
||||||
|
require.NoError(t, markRenameToken(entry, clientToken, renameSource, "dst.txt"))
|
||||||
|
|
||||||
|
moved := proto.Clone(entry).(*filer_pb.Entry)
|
||||||
|
moved.Name = "dst.txt"
|
||||||
|
|
||||||
|
assert.Equal(t, renameTokenSameRequest, classifyRenameToken(moved, clientToken, renameSource, "dst.txt"))
|
||||||
|
assert.Equal(t, renameTokenReused, classifyRenameToken(moved, clientToken, "/bucket/other.txt", "dst.txt"))
|
||||||
|
assert.Equal(t, renameTokenUnrelated, classifyRenameToken(moved, "rename-token-of-another-request", renameSource, "dst.txt"))
|
||||||
|
// A copy of a renamed object carries its token along; the key it names does not.
|
||||||
|
assert.Equal(t, renameTokenUnrelated, classifyRenameToken(moved, clientToken, renameSource, "copy.txt"))
|
||||||
|
|
||||||
|
// The token is the gateway's own bookkeeping and must not reach a GET or HEAD.
|
||||||
|
assert.True(t, s3_constants.IsSeaweedFSInternalHeader(s3_constants.SeaweedFSRenameToken))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A reused token is refused with 400 Bad Request, matching the AWS S3
|
||||||
|
// RenameObject API documentation for IdempotencyParameterMismatch.
|
||||||
|
func TestRenameTokenReuseAnswersBadRequest(t *testing.T) {
|
||||||
|
api := s3err.GetAPIError(s3err.ErrIdempotentParameterMismatch)
|
||||||
|
assert.Equal(t, http.StatusBadRequest, api.HTTPStatusCode)
|
||||||
|
assert.Equal(t, "IdempotentParameterMismatch", api.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClassifyRenameToken(t *testing.T) {
|
||||||
|
const clientToken = "rename-token-of-this-request"
|
||||||
|
const renameSource = "/bucket/src.txt"
|
||||||
|
|
||||||
|
stamped := func(token renameToken) *filer_pb.Entry {
|
||||||
|
raw, err := json.Marshal(token)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return &filer_pb.Entry{Name: "dst.txt", Extended: map[string][]byte{s3_constants.SeaweedFSRenameToken: raw}}
|
||||||
|
}
|
||||||
|
live := renameToken{Token: clientToken, Source: renameSource, Dest: "dst.txt", Unix: time.Now().Unix()}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
dstEntry *filer_pb.Entry
|
||||||
|
clientToken string
|
||||||
|
want renameTokenVerdict
|
||||||
|
}{
|
||||||
|
{"nothing at the destination", nil, clientToken, renameTokenUnrelated},
|
||||||
|
{"request without a token", stamped(live), "", renameTokenUnrelated},
|
||||||
|
{"destination the rename never wrote", &filer_pb.Entry{Name: "dst.txt"}, clientToken, renameTokenUnrelated},
|
||||||
|
{"another rename's token", stamped(live), "rename-token-of-another-request", renameTokenUnrelated},
|
||||||
|
{"same request", stamped(live), clientToken, renameTokenSameRequest},
|
||||||
|
{"token reused for another source", stamped(renameToken{Token: clientToken, Source: "/bucket/other.txt", Dest: "dst.txt", Unix: time.Now().Unix()}), clientToken, renameTokenReused},
|
||||||
|
{"token of a rename to another key", stamped(renameToken{Token: clientToken, Source: renameSource, Dest: "elsewhere.txt", Unix: time.Now().Unix()}), clientToken, renameTokenUnrelated},
|
||||||
|
{"expired token", stamped(renameToken{Token: clientToken, Source: renameSource, Dest: "dst.txt", Unix: time.Now().Add(-renameTokenValidity - time.Minute).Unix()}), clientToken, renameTokenUnrelated},
|
||||||
|
{"unreadable token", &filer_pb.Entry{Name: "dst.txt", Extended: map[string][]byte{s3_constants.SeaweedFSRenameToken: []byte("{")}}, clientToken, renameTokenUnrelated},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
assert.Equal(t, tc.want, classifyRenameToken(tc.dstEntry, tc.clientToken, renameSource, "dst.txt"))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRetryRenameDecision covers the handler's retry branch, including the
|
||||||
|
// fallthrough the classification alone cannot express: when the destination
|
||||||
|
// carries this request's token but the source is still there, the rename is
|
||||||
|
// performed rather than short-circuited.
|
||||||
|
func TestRetryRenameDecision(t *testing.T) {
|
||||||
|
const clientToken = "rename-token-of-this-request"
|
||||||
|
const renameSource = "/bucket/src.txt"
|
||||||
|
|
||||||
|
stamped := func(token renameToken) *filer_pb.Entry {
|
||||||
|
raw, err := json.Marshal(token)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return &filer_pb.Entry{Name: "dst.txt", Extended: map[string][]byte{s3_constants.SeaweedFSRenameToken: raw}}
|
||||||
|
}
|
||||||
|
live := renameToken{Token: clientToken, Source: renameSource, Dest: "dst.txt", Unix: time.Now().Unix()}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
dstEntry *filer_pb.Entry
|
||||||
|
clientToken string
|
||||||
|
srcErrCode s3err.ErrorCode
|
||||||
|
wantErr s3err.ErrorCode
|
||||||
|
wantSettled bool
|
||||||
|
}{
|
||||||
|
// A retry whose source is gone is answered as success.
|
||||||
|
{"same request, source gone", stamped(live), clientToken, s3err.ErrNoSuchKey, s3err.ErrNone, true},
|
||||||
|
// A retry whose source is back falls through: the rename proceeds.
|
||||||
|
{"same request, source present", stamped(live), clientToken, s3err.ErrNone, s3err.ErrNone, false},
|
||||||
|
// A reused token is refused regardless of source state.
|
||||||
|
{"reused token, source gone", stamped(renameToken{Token: clientToken, Source: "/bucket/other.txt", Dest: "dst.txt", Unix: time.Now().Unix()}), clientToken, s3err.ErrNoSuchKey, s3err.ErrIdempotentParameterMismatch, true},
|
||||||
|
{"reused token, source present", stamped(renameToken{Token: clientToken, Source: "/bucket/other.txt", Dest: "dst.txt", Unix: time.Now().Unix()}), clientToken, s3err.ErrNone, s3err.ErrIdempotentParameterMismatch, true},
|
||||||
|
// An unrelated destination falls through to the ordinary rename path.
|
||||||
|
{"unrelated, source gone", nil, clientToken, s3err.ErrNoSuchKey, s3err.ErrNone, false},
|
||||||
|
{"unrelated, source present", nil, clientToken, s3err.ErrNone, s3err.ErrNone, false},
|
||||||
|
// A request without a token is never settled by the retry path.
|
||||||
|
{"no token, destination stamped", stamped(live), "", s3err.ErrNoSuchKey, s3err.ErrNone, false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
errCode, settled := retryRenameDecision(tc.dstEntry, tc.clientToken, renameSource, "dst.txt", tc.srcErrCode)
|
||||||
|
assert.Equal(t, tc.wantSettled, settled)
|
||||||
|
assert.Equal(t, tc.wantErr, errCode)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestRouting_RenameObject pins PUT /bucket/key?renameObject to the RenameObject
|
// TestRouting_RenameObject pins PUT /bucket/key?renameObject to the RenameObject
|
||||||
// route rather than the plain PutObject one that would otherwise match it.
|
// route rather than the plain PutObject one that would otherwise match it.
|
||||||
func TestRouting_RenameObject(t *testing.T) {
|
func TestRouting_RenameObject(t *testing.T) {
|
||||||
|
|||||||
@@ -170,6 +170,7 @@ const (
|
|||||||
|
|
||||||
ErrInvalidRenameSource
|
ErrInvalidRenameSource
|
||||||
ErrRenameDestinationSameAsSource
|
ErrRenameDestinationSameAsSource
|
||||||
|
ErrIdempotentParameterMismatch
|
||||||
)
|
)
|
||||||
|
|
||||||
// Error message constants for checksum validation
|
// Error message constants for checksum validation
|
||||||
@@ -381,6 +382,13 @@ var errorCodeResponse = map[ErrorCode]APIError{
|
|||||||
Description: "This rename request is illegal because it is trying to rename an object to itself.",
|
Description: "This rename request is illegal because it is trying to rename an object to itself.",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrIdempotentParameterMismatch: {
|
||||||
|
Code: "IdempotentParameterMismatch",
|
||||||
|
Description: "The request uses the same client token as a previous, but non-identical request.",
|
||||||
|
// 400 Bad Request, matching the AWS S3 RenameObject API documentation
|
||||||
|
// for IdempotencyParameterMismatch.
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrInvalidTag: {
|
ErrInvalidTag: {
|
||||||
Code: "InvalidTag",
|
Code: "InvalidTag",
|
||||||
Description: "The Tag value you have provided is invalid",
|
Description: "The Tag value you have provided is invalid",
|
||||||
|
|||||||
Reference in New Issue
Block a user