From 2d41d3715026a4f9d180199297e7fbc23ec29676 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 23 May 2026 09:07:02 -0700 Subject: [PATCH] s3: route non-versioned CompleteMultipartUpload off the DLM A non-versioned multipart completion's only contended mutation is the final object CreateEntry; assembling parts and the idempotent-replay check are read-only. So when the destination is non-versioned, route that final write to the object's owner with the precondition (reusing the route-by-key path) and run the assembly outside any lock, skipping the distributed lock and gateway precondition. mkFileRouted builds the object entry the way mkFile does, applies the existing modifier, and routes the conditioned CreateEntry to the owner. Versioned and suspended completions keep the lock path. This is a cold path (one completion per upload), so the win is consistency with the other write paths rather than throughput. --- weed/s3api/filer_multipart.go | 37 ++++++++++++++++--- weed/s3api/s3api_object_routed_write.go | 49 +++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 6a113f9e4..126a91589 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -30,6 +30,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" ) @@ -553,9 +554,16 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId entryName, dirName := s3a.getEntryNameAndDir(input) var completionState *multipartCompletionState - finalizeCode := s3a.withObjectWriteLock(*input.Bucket, *input.Key, func() s3err.ErrorCode { - return s3a.checkConditionalHeaders(r, *input.Bucket, *input.Key) - }, func() s3err.ErrorCode { + // Non-versioned destination: the final write is a single CreateEntry, so route + // it (with the precondition) to the object's owner and skip the distributed + // lock. Assembly and the idempotency replay below are read-only and safe + // outside any lock. Versioned/suspended completions keep the lock path. + mpuCond, mpuCondOk := buildWriteCondition(r) + var nvOwner pb.ServerAddress + if vs, vErr := s3a.getVersioningState(*input.Bucket); vErr == nil && vs == "" && mpuCondOk { + nvOwner, _ = s3a.routedObjectOwner(*input.Bucket, *input.Key) + } + finalizeBody := func() s3err.ErrorCode { var prepCode s3err.ErrorCode completionState, output, prepCode = s3a.prepareMultipartCompletionState(r, input, uploadDirectory, entryName, dirName, completedPartNumbers, completedPartMap, maxPartNo) if prepCode != s3err.ErrNone || output != nil { @@ -739,7 +747,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl } // For non-versioned buckets, create main object file - if err := s3a.mkFile(dirName, entryName, completionState.finalParts, func(entry *filer_pb.Entry) { + nvModifier := func(entry *filer_pb.Entry) { if entry.Extended == nil { entry.Extended = make(map[string][]byte) } @@ -787,7 +795,14 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl if completionState.entityWithTtl { entry.Extended[s3_constants.SeaweedFSExpiresS3] = []byte("true") } - }); err != nil { + } + if nvOwner != "" { + // Routed path: the owner evaluates the precondition and writes under + // its local lock, so the lock wrapper below is skipped. + if code := s3a.mkFileRouted(nvOwner, dirName, entryName, completionState.finalParts, mpuCond, nvModifier); code != s3err.ErrNone { + return code + } + } else if err := s3a.mkFile(dirName, entryName, completionState.finalParts, nvModifier); err != nil { glog.Errorf("completeMultipartUpload %s/%s error: %v", dirName, entryName, err) return s3err.ErrInternalError } @@ -802,7 +817,17 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl ChecksumValue: completionState.checksumValue, } return s3err.ErrNone - }) + } + var finalizeCode s3err.ErrorCode + if nvOwner != "" { + // Routed non-versioned completion: no distributed lock, no gateway + // precondition here — the routed write carries the condition. + finalizeCode = finalizeBody() + } else { + finalizeCode = s3a.withObjectWriteLock(*input.Bucket, *input.Key, func() s3err.ErrorCode { + return s3a.checkConditionalHeaders(r, *input.Bucket, *input.Key) + }, finalizeBody) + } if finalizeCode != s3err.ErrNone { return nil, finalizeCode } diff --git a/weed/s3api/s3api_object_routed_write.go b/weed/s3api/s3api_object_routed_write.go index f9e354f16..1273f7aab 100644 --- a/weed/s3api/s3api_object_routed_write.go +++ b/weed/s3api/s3api_object_routed_write.go @@ -5,7 +5,9 @@ import ( "fmt" "net/http" "strings" + "time" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" @@ -115,6 +117,53 @@ func singleStrongETag(v string) (string, bool) { return strings.Trim(v, `"`), true } +// mkFileRouted builds a file entry the way filer_pb.MkFile does, applies fn, and +// routes the CreateEntry (with an optional precondition) to the given owner filer +// so its local per-path lock serializes the write. Used by callers that build an +// object via a modifier (e.g. multipart completion) but want the routed, +// distributed-lock-free write path. +func (s3a *S3ApiServer) mkFileRouted(owner pb.ServerAddress, dir, name string, chunks []*filer_pb.FileChunk, cond *filer_pb.WriteCondition, fn func(*filer_pb.Entry)) s3err.ErrorCode { + now := time.Now().Unix() + entry := &filer_pb.Entry{ + Name: name, + IsDirectory: false, + Attributes: &filer_pb.FuseAttributes{ + Mtime: now, + Crtime: now, + FileMode: uint32(0770), + Uid: filer_pb.OS_UID, + Gid: filer_pb.OS_GID, + }, + Chunks: chunks, + } + if fn != nil { + fn(entry) + } + resp, err := s3a.createEntryOnFiler(owner, &filer_pb.CreateEntryRequest{ + Directory: dir, + Entry: entry, + Condition: cond, + }) + switch { + case err != nil: + glog.Errorf("mkFileRouted: %s/%s on %s: %v", dir, name, owner, err) + return s3err.ErrInternalError + case resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED: + return s3err.ErrPreconditionFailed + case resp.ErrorCode != filer_pb.FilerError_OK: + if code, ok := filerErrorCodeToS3Error(resp.ErrorCode); ok { + return code + } + glog.Errorf("mkFileRouted: %s/%s unexpected code %v", dir, name, resp.ErrorCode) + return s3err.ErrInternalError + case resp.Error != "": + glog.Errorf("mkFileRouted: %s/%s: %s", dir, name, resp.Error) + return s3err.ErrInternalError + default: + return s3err.ErrNone + } +} + // createEntryOnFiler sends a CreateEntry directly to the given owner filer so // its local per-path lock serializes the write. The raw response is returned so // the caller can distinguish PRECONDITION_FAILED from other outcomes.