diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 76c1086aab..cb1e012e3d 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -95,7 +95,7 @@ func (s3a *S3ApiServer) createMultipartUpload(r *http.Request, input *s3.CreateM // Prepare error handling outside callback scope var encryptionError error - if err := s3a.mkdir(s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) { + if err := s3a.uploadFiler(*input.Bucket, *input.Key).mkdir(s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) { if entry.Extended == nil { entry.Extended = make(map[string][]byte) } @@ -388,14 +388,14 @@ func applyMultipartSSES3HeadersFromUploadEntry(dst *filer_pb.Entry, sses3Info *m func (s3a *S3ApiServer) prepareMultipartCompletionState(r *http.Request, input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload, uploadDirectory, entryName, dirName string, completedPartNumbers []int, completedPartMap map[int][]string, maxPartNo int) (*multipartCompletionState, *CompleteMultipartUploadResult, s3err.ErrorCode) { if output, _ := s3a.committedCompletion(r, input); output != nil { - cleanupEntries, _, cleanupErr := s3a.list(uploadDirectory, "", "", false, s3_constants.MaxS3MultipartParts+1) + cleanupEntries, _, cleanupErr := s3a.uploadFiler(*input.Bucket, *input.Key).list(uploadDirectory, "", "", false, s3_constants.MaxS3MultipartParts+1) if cleanupErr != nil && !errors.Is(cleanupErr, filer_pb.ErrNotFound) { glog.Warningf("completeMultipartUpload: failed to list stale upload directory %s for cleanup: %v", uploadDirectory, cleanupErr) } return &multipartCompletionState{deleteEntries: cleanupEntries, metadataOnlyCleanup: true}, output, s3err.ErrNone } - entries, _, err := s3a.list(uploadDirectory, "", "", false, s3_constants.MaxS3MultipartParts+1) + entries, _, err := s3a.uploadFiler(*input.Bucket, *input.Key).list(uploadDirectory, "", "", false, s3_constants.MaxS3MultipartParts+1) if err != nil { glog.Errorf("completeMultipartUpload %s %s error: %v", *input.Bucket, *input.UploadId, err) if isFilerNotFound(err) { @@ -416,7 +416,7 @@ func (s3a *S3ApiServer) prepareMultipartCompletionState(r *http.Request, input * return nil, nil, s3err.ErrNoSuchUpload } - pentry, err := s3a.getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId) + pentry, err := s3a.uploadFiler(*input.Bucket, *input.Key).getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId) if err != nil { glog.Errorf("completeMultipartUpload %s %s error: %v", *input.Bucket, *input.UploadId, err) if errors.Is(err, filer_pb.ErrNotFound) { @@ -796,7 +796,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl // The transaction removes the upload directory metadata-only, // so the part entries the object does not reference are freed // first or their chunks leak. - if err := s3a.deleteUnusedPartEntries(r.Context(), uploadDirectory, *input.Bucket, *input.UploadId, completionState); err != nil { + if err := s3a.deleteUnusedPartEntries(r.Context(), uploadDirectory, *input.Bucket, *input.Key, *input.UploadId, completionState); err != nil { glog.Errorf("completeMultipartUpload %s upload %s unused part cleanup: %v", *input.Bucket, *input.UploadId, err) return s3err.ErrInternalError } @@ -807,7 +807,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl // Roll back only while the upload directory survives: once the // transaction removed it, the version file is the only record // left and deleting it would make a retry impossible. - if exists, _ := s3a.exists(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true); exists { + if exists, _ := s3a.uploadFiler(*input.Bucket, *input.Key).exists(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true); exists { if rollbackErr := s3a.rollbackMultipartVersion(versionDir, versionFileName); rollbackErr != nil { glog.Errorf("completeMultipartUpload: failed to rollback version %s for %s/%s after routed finalize error: %v", versionId, *input.Bucket, *input.Key, rollbackErr) completionState.manifestsReferenced = true @@ -849,7 +849,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl if versioningState == s3_constants.VersioningSuspended { // For suspended versioning, add "null" version ID metadata and return "null" version ID - removal, err := s3a.routedUploadRemoval(r.Context(), owner, uploadDirectory, *input.Bucket, *input.UploadId, completionState) + removal, err := s3a.routedUploadRemoval(r.Context(), owner, uploadDirectory, *input.Bucket, *input.Key, *input.UploadId, completionState) if err != nil { glog.Errorf("completeMultipartUpload %s upload %s unused part cleanup: %v", *input.Bucket, *input.UploadId, err) return s3err.ErrInternalError @@ -945,7 +945,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl } // For non-versioned buckets, create main object file - removal, err := s3a.routedUploadRemoval(r.Context(), owner, uploadDirectory, *input.Bucket, *input.UploadId, completionState) + removal, err := s3a.routedUploadRemoval(r.Context(), owner, uploadDirectory, *input.Bucket, *input.Key, *input.UploadId, completionState) if err != nil { glog.Errorf("completeMultipartUpload %s upload %s unused part cleanup: %v", *input.Bucket, *input.UploadId, err) return s3err.ErrInternalError @@ -1073,9 +1073,9 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl cleanupCtx := withFilerRetryBudget(context.Background(), filerRetryRequestBudget) // Keep the directory when an entry delete failed so its metadata // still references the chunks; removing it metadata-only orphans them. - if err := s3a.deleteUnusedPartEntries(cleanupCtx, uploadDirectory, *input.Bucket, *input.UploadId, completionState); err != nil { + if err := s3a.deleteUnusedPartEntries(cleanupCtx, uploadDirectory, *input.Bucket, *input.Key, *input.UploadId, completionState); err != nil { glog.V(1).Infof("completeMultipartUpload cleanup %s upload %s: %v", *input.Bucket, *input.UploadId, err) - } else if err := s3a.rm(cleanupCtx, s3a.genUploadsFolder(*input.Bucket), *input.UploadId, false, true); err != nil { + } else if err := s3a.uploadFiler(*input.Bucket, *input.Key).rm(cleanupCtx, s3a.genUploadsFolder(*input.Bucket), *input.UploadId, false, true); err != nil { glog.V(1).Infof("completeMultipartUpload cleanup %s upload %s: %v", *input.Bucket, *input.UploadId, err) } } @@ -1091,10 +1091,10 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl // reference. A finalize that removes the whole upload directory metadata-only // frees them first, or their chunks leak; a failure here must abort the // completion, since the surviving entries keep the upload retriable. -func (s3a *S3ApiServer) deleteUnusedPartEntries(ctx context.Context, uploadDirectory, bucket, uploadId string, completionState *multipartCompletionState) error { +func (s3a *S3ApiServer) deleteUnusedPartEntries(ctx context.Context, uploadDirectory, bucket, object, uploadId string, completionState *multipartCompletionState) error { var lastErr error for _, deleteEntry := range completionState.deleteEntries { - if err := s3a.rm(ctx, uploadDirectory, deleteEntry.Name, !completionState.metadataOnlyCleanup, true); err != nil { + if err := s3a.uploadFiler(bucket, object).rm(ctx, uploadDirectory, deleteEntry.Name, !completionState.metadataOnlyCleanup, true); err != nil { glog.Warningf("completeMultipartUpload cleanup %s upload %s unused %s : %v", bucket, uploadId, deleteEntry.Name, err) lastErr = err } @@ -1182,7 +1182,7 @@ func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput glog.V(2).Infof("abortMultipartUpload input %v", input) uploadsFolder := s3a.genUploadsFolder(*input.Bucket) - uploadEntry, err := s3a.getEntry(uploadsFolder, *input.UploadId) + uploadEntry, err := s3a.uploadFiler(*input.Bucket, aws.StringValue(input.Key)).getEntry(uploadsFolder, *input.UploadId) // complete and abort both remove .uploads/; a missing record is an upload // that does not exist (NoSuchUpload on AWS), not one this call discarded if err != nil { @@ -1215,7 +1215,7 @@ func (s3a *S3ApiServer) removeUploadDir(bucket, uploadId, object string) s3err.E return s3err.ErrInternalError } if completed { - if err := s3a.rm(context.Background(), s3a.genUploadsFolder(bucket), uploadId, false, true); err != nil { + if err := s3a.uploadFiler(bucket, object).rm(context.Background(), s3a.genUploadsFolder(bucket), uploadId, false, true); err != nil { glog.V(1).Infof("bucket %s remove upload %s: %v", bucket, uploadId, err) return s3err.ErrInternalError } @@ -1233,7 +1233,7 @@ func (s3a *S3ApiServer) removeUploadDir(bucket, uploadId, object string) s3err.E func (s3a *S3ApiServer) routedUploadDirDelete(bucket, uploadId, object string) s3err.ErrorCode { uploadsFolder := s3a.genUploadsFolder(bucket) rmUploadDir := func(isDeleteData bool) s3err.ErrorCode { - if err := s3a.rm(context.Background(), uploadsFolder, uploadId, isDeleteData, true); err != nil { + if err := s3a.uploadFiler(bucket, object).rm(context.Background(), uploadsFolder, uploadId, isDeleteData, true); err != nil { glog.V(1).Infof("bucket %s remove upload %s: %v", bucket, uploadId, err) return s3err.ErrInternalError } @@ -1588,7 +1588,7 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP // directory as empty instead of erroring, so listing alone cannot tell a // completed upload (NoSuchUpload on AWS) from an open upload with no parts // yet (200 with an empty list). Probe the upload record instead. - pentry, err := s3a.getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId) + pentry, err := s3a.uploadFiler(*input.Bucket, *input.Key).getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId) if err != nil { if errors.Is(err, filer_pb.ErrNotFound) { return nil, s3err.ErrNoSuchUpload @@ -1618,7 +1618,7 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP output.IsTruncated = aws.Bool(false) return output, s3err.ErrNone } - entries, isLast, err := s3a.list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d", *input.PartNumberMarker+1), true, uint32(*input.MaxParts)) + entries, isLast, err := s3a.uploadFiler(*input.Bucket, *input.Key).list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d", *input.PartNumberMarker+1), true, uint32(*input.MaxParts)) if err != nil { // A store that reports the missing upload directory as not-found means // the upload is gone (completed or aborted), not a store error. diff --git a/weed/s3api/s3api_bucket_default_encryption_test.go b/weed/s3api/s3api_bucket_default_encryption_test.go index 1ac7b19447..8815ce5a09 100644 --- a/weed/s3api/s3api_bucket_default_encryption_test.go +++ b/weed/s3api/s3api_bucket_default_encryption_test.go @@ -138,7 +138,7 @@ func TestPutObjectPartSkipsBucketDefaultEncryption(t *testing.T) { }) filePath := "/buckets/b/.uploads/up1/0002_deadbeef.part" - r := withMultipartPart(httptest.NewRequest(http.MethodPut, "/b/o?partNumber=2&uploadId=up1", nil)) + r := withMultipartPart(httptest.NewRequest(http.MethodPut, "/b/o?partNumber=2&uploadId=up1", nil), s3a.uploadFiler("b", "o")) etag, code, sseMeta := s3a.putToFiler(r, filePath, strings.NewReader("part data"), "b", "", 2, 0, nil, false, "") if code != s3err.ErrNone { t.Fatalf("putToFiler = %v, want %v", code, s3err.ErrNone) diff --git a/weed/s3api/s3api_bucket_policy_engine.go b/weed/s3api/s3api_bucket_policy_engine.go index 6f92dfdb42..b2708c486e 100644 --- a/weed/s3api/s3api_bucket_policy_engine.go +++ b/weed/s3api/s3api_bucket_policy_engine.go @@ -17,7 +17,7 @@ type BucketPolicyEngine struct { // "aws:kms") that was stored when a multipart upload was initiated. // Returns "" when the upload had no SSE or when the entry cannot be found. // Set by S3ApiServer after construction to give the engine filer access. - MultipartSSELookup func(bucket, uploadID string) string + MultipartSSELookup func(bucket, object, uploadID string) string } // NewBucketPolicyEngine creates a new bucket policy engine @@ -169,7 +169,7 @@ func (bpe *BucketPolicyEngine) EvaluatePolicy(bucket, object, action, principal // policy conditions on s3:x-amz-server-side-encryption evaluate correctly. if policy_engine.IsMultipartContinuationAction(s3Action) && bpe.MultipartSSELookup != nil { if uploadID := r.URL.Query().Get("uploadId"); uploadID != "" { - args.InheritedSSEAlgorithm = bpe.MultipartSSELookup(bucket, uploadID) + args.InheritedSSEAlgorithm = bpe.MultipartSSELookup(bucket, object, uploadID) } } } diff --git a/weed/s3api/s3api_copy_source_classify_test.go b/weed/s3api/s3api_copy_source_classify_test.go index 609baf32e0..1103268abb 100644 --- a/weed/s3api/s3api_copy_source_classify_test.go +++ b/weed/s3api/s3api_copy_source_classify_test.go @@ -65,12 +65,12 @@ func TestCopyAfterRoutedPut(t *testing.T) { var uploadID string if multipart { uploadID = s3a.generateUploadID("final/object") - replica.mu.Lock() - replica.entries["/buckets/destination/.uploads/"+uploadID] = &filer_pb.Entry{ + owner.mu.Lock() + owner.entries["/buckets/destination/.uploads/"+uploadID] = &filer_pb.Entry{ Name: uploadID, IsDirectory: true, Extended: map[string][]byte{s3_constants.ExtMultipartObjectKey: []byte("final/object")}, } - replica.mu.Unlock() + owner.mu.Unlock() copyURL += "?partNumber=1&uploadId=" + uploadID } copyRequest := httptest.NewRequest(http.MethodPut, copyURL, nil) @@ -87,15 +87,17 @@ func TestCopyAfterRoutedPut(t *testing.T) { replica.mu.Lock() if multipart { + replica.mu.Unlock() + owner.mu.Lock() var part *filer_pb.Entry - for key, entry := range replica.entries { + for key, entry := range owner.entries { if strings.HasPrefix(key, "/buckets/destination/.uploads/"+uploadID+"/") && !entry.IsDirectory { part = entry } } - require.NotNil(t, part, "copy must persist the part") + owner.mu.Unlock() + require.NotNil(t, part, "copy must persist the part next to the upload") require.Equal(t, stored.Attributes.Md5, part.Attributes.Md5) - replica.mu.Unlock() } else { // The destination commits on its write owner, the same filer a // later copy resolves it from before replication. diff --git a/weed/s3api/s3api_object_handlers_copy.go b/weed/s3api/s3api_object_handlers_copy.go index 10d81a8a64..1b67c7bfd6 100644 --- a/weed/s3api/s3api_object_handlers_copy.go +++ b/weed/s3api/s3api_object_handlers_copy.go @@ -1181,7 +1181,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req // uploadEntry=nil would silently skip the SSE check on the destination // side and could send a plain-source copy through the raw-byte fast // path even though the destination's encryption state is unknown. - uploadEntry, uploadEntryErr := s3a.getEntry(s3a.genUploadsFolder(dstBucket), uploadID) + uploadEntry, uploadEntryErr := s3a.uploadFiler(dstBucket, dstObject).getEntry(s3a.genUploadsFolder(dstBucket), uploadID) if uploadEntryErr != nil { if errors.Is(uploadEntryErr, filer_pb.ErrNotFound) { s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) @@ -1285,14 +1285,15 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req // Save the part entry to the multipart uploads folder // Check if part exists and remove it first (allow re-copying same part) - if exists, _ := s3a.exists(uploadDir, partName, false); exists { - if err := s3a.rm(r.Context(), uploadDir, partName, false, false); err != nil { + upload := s3a.uploadFiler(dstBucket, dstObject) + if exists, _ := upload.exists(uploadDir, partName, false); exists { + if err := upload.rm(r.Context(), uploadDir, partName, false, false); err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } } - if err := s3a.mkFile(uploadDir, partName, dstEntry.Chunks, func(entry *filer_pb.Entry) { + if err := upload.mkFile(uploadDir, partName, dstEntry.Chunks, func(entry *filer_pb.Entry) { entry.Attributes = dstEntry.Attributes entry.Extended = dstEntry.Extended }); err != nil { diff --git a/weed/s3api/s3api_object_handlers_copy_etag_test.go b/weed/s3api/s3api_object_handlers_copy_etag_test.go index a6ec92bfac..bdc557adab 100644 --- a/weed/s3api/s3api_object_handlers_copy_etag_test.go +++ b/weed/s3api/s3api_object_handlers_copy_etag_test.go @@ -442,7 +442,7 @@ func TestWriteEmptyCopyPartStoresMd5(t *testing.T) { s3a := newPutTestServer(t, startFakeFiler(t, filerImpl)) util_http.InitGlobalHttpClient() - etag, code := s3a.writeEmptyCopyPart("b", "u1", 3) + etag, code := s3a.writeEmptyCopyPart("b", "dst", "u1", 3) if code != s3err.ErrNone { t.Fatalf("writeEmptyCopyPart: %v", code) } diff --git a/weed/s3api/s3api_object_handlers_copy_part_sse.go b/weed/s3api/s3api_object_handlers_copy_part_sse.go index e6c0304bc4..1fda5dd40e 100644 --- a/weed/s3api/s3api_object_handlers_copy_part_sse.go +++ b/weed/s3api/s3api_object_handlers_copy_part_sse.go @@ -374,7 +374,7 @@ func (s3a *S3ApiServer) copyObjectPartViaReencryption( uploadEntry *filer_pb.Entry, ) (etag string, sseMetadata SSEResponseMetadata, errCode s3err.ErrorCode) { if endOffset < startOffset && !uploadEntryHasChecksum(uploadEntry) { - tag, code := s3a.writeEmptyCopyPart(dstBucket, uploadID, partID) + tag, code := s3a.writeEmptyCopyPart(dstBucket, dstObject, uploadID, partID) return tag, SSEResponseMetadata{}, code } sliceLen := int64(0) @@ -404,7 +404,7 @@ func (s3a *S3ApiServer) copyObjectPartViaReencryption( applyDestChecksumHeaderToCopyRequest(cloned, uploadEntry) filePath := s3a.genPartUploadPath(dstBucket, uploadID, partID) - tag, code, putSSE := s3a.putToFiler(withMultipartPart(cloned), filePath, srcReader, dstBucket, "", partID, 0, nil, false, s3a.toFilerPath(dstBucket, dstObject)) + tag, code, putSSE := s3a.putToFiler(withMultipartPart(cloned, s3a.uploadFiler(dstBucket, dstObject)), filePath, srcReader, dstBucket, "", partID, 0, nil, false, s3a.toFilerPath(dstBucket, dstObject)) if code != s3err.ErrNone { return "", SSEResponseMetadata{}, code } @@ -413,15 +413,16 @@ func (s3a *S3ApiServer) copyObjectPartViaReencryption( // writeEmptyCopyPart writes a 0-byte part entry for an empty UploadPartCopy // range, mirroring the legacy fast path's handling of endOffset < startOffset. -func (s3a *S3ApiServer) writeEmptyCopyPart(dstBucket, uploadID string, partID int) (string, s3err.ErrorCode) { +func (s3a *S3ApiServer) writeEmptyCopyPart(dstBucket, dstObject, uploadID string, partID int) (string, s3err.ErrorCode) { uploadDir, partName := s3a.copyPartLocation(dstBucket, uploadID, partID) - if exists, _ := s3a.exists(uploadDir, partName, false); exists { - if err := s3a.rm(context.Background(), uploadDir, partName, false, false); err != nil { + upload := s3a.uploadFiler(dstBucket, dstObject) + if exists, _ := upload.exists(uploadDir, partName, false); exists { + if err := upload.rm(context.Background(), uploadDir, partName, false, false); err != nil { return "", s3err.ErrInternalError } } const emptyMD5Hex = "d41d8cd98f00b204e9800998ecf8427e" - if err := s3a.mkFile(uploadDir, partName, nil, func(e *filer_pb.Entry) { + if err := upload.mkFile(uploadDir, partName, nil, func(e *filer_pb.Entry) { if e.Attributes == nil { e.Attributes = &filer_pb.FuseAttributes{} } diff --git a/weed/s3api/s3api_object_handlers_multipart.go b/weed/s3api/s3api_object_handlers_multipart.go index d2a0455081..1aa504704e 100644 --- a/weed/s3api/s3api_object_handlers_multipart.go +++ b/weed/s3api/s3api_object_handlers_multipart.go @@ -413,7 +413,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ glog.V(2).Infof("PutObjectPartHandler %s %s %04d", bucket, uploadID, partID) // Verify the multipart upload exists (rejects parts after abort) - uploadEntry, err := s3a.getEntry(s3a.genUploadsFolder(bucket), uploadID) + uploadEntry, err := s3a.uploadFiler(bucket, object).getEntry(s3a.genUploadsFolder(bucket), uploadID) if errors.Is(err, filer_pb.ErrNotFound) { s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) return @@ -514,7 +514,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ // start the TTL clock before CompleteMultipartUpload ever assembled // the object. filer.conf storage rules are the opposite case: they // place the bytes the part becomes, so they resolve against the object. - etag, errCode, sseMetadata := s3a.putToFiler(withMultipartPart(r), filePath, dataReader, bucket, "", partID, 0, nil, false, s3a.toFilerPath(bucket, object)) + etag, errCode, sseMetadata := s3a.putToFiler(withMultipartPart(r, s3a.uploadFiler(bucket, object)), filePath, dataReader, bucket, "", partID, 0, nil, false, s3a.toFilerPath(bucket, object)) if errCode != s3err.ErrNone { glog.Errorf("PutObjectPart: putToFiler failed with error code %v for bucket=%s, object=%s, partNumber=%d", errCode, bucket, object, partID) @@ -554,7 +554,7 @@ func isMultipartUploadEntry(entry *filer_pb.Entry) bool { // part along with the directory it resurrected when an abort was answered while // the part was in flight. It reports whether the caller may answer success. func (s3a *S3ApiServer) checkUploadStillOpen(w http.ResponseWriter, r *http.Request, bucket, object, uploadID string) bool { - entry, err := s3a.getEntry(s3a.genUploadsFolder(bucket), uploadID) + entry, err := s3a.uploadFiler(bucket, object).getEntry(s3a.genUploadsFolder(bucket), uploadID) if err != nil && !errors.Is(err, filer_pb.ErrNotFound) { glog.Errorf("checkUploadStillOpen %s/%s: %v", bucket, uploadID, err) s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) @@ -580,8 +580,8 @@ func (s3a *S3ApiServer) checkUploadStillOpen(w http.ResponseWriter, r *http.Requ // if the upload entry is not found or had no SSE. It is used by the bucket // policy engine to evaluate s3:x-amz-server-side-encryption conditions for // UploadPart and UploadPartCopy, which do not re-send the SSE header. -func (s3a *S3ApiServer) getMultipartSSEAlgorithm(bucket, uploadID string) string { - entry, err := s3a.getEntry(s3a.genUploadsFolder(bucket), uploadID) +func (s3a *S3ApiServer) getMultipartSSEAlgorithm(bucket, object, uploadID string) string { + entry, err := s3a.uploadFiler(bucket, object).getEntry(s3a.genUploadsFolder(bucket), uploadID) if err != nil || entry == nil || entry.Extended == nil { return "" } diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index 8ecbb04c16..cea0ef884b 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -540,13 +540,14 @@ func (s3a *S3ApiServer) uploadChunkSize() int32 { } // multipartPartContextKey marks a putToFiler request as writing a staged -// multipart part. Encryption for the upload is decided at -// CreateMultipartUpload and replayed through request headers, so bucket -// default encryption must not be re-evaluated for part writes. +// multipart part, carrying the filer the upload's directory lives on. +// Encryption for the upload is decided at CreateMultipartUpload and replayed +// through request headers, so bucket default encryption must not be +// re-evaluated for part writes. type multipartPartContextKey struct{} -func withMultipartPart(r *http.Request) *http.Request { - return r.WithContext(context.WithValue(r.Context(), multipartPartContextKey{}, true)) +func withMultipartPart(r *http.Request, upload uploadFilerClient) *http.Request { + return r.WithContext(context.WithValue(r.Context(), multipartPartContextKey{}, upload)) } // validatePutHeaders checks x-amz-storage-class and x-amz-tagging, so the @@ -572,10 +573,15 @@ func validatePutHeaders(r *http.Request) (map[string]string, s3err.ErrorCode) { } func isMultipartPartRequest(r *http.Request) bool { - marked, _ := r.Context().Value(multipartPartContextKey{}).(bool) + _, marked := multipartPartUpload(r) return marked } +func multipartPartUpload(r *http.Request) (uploadFilerClient, bool) { + upload, marked := r.Context().Value(multipartPartContextKey{}).(uploadFilerClient) + return upload, marked +} + // putToFiler writes one chunk of object bytes (a full PutObject body, a // single MPU part, a copy-part destination). lifecycleTTLSec is non-zero // only for top-level PutObject paths where the lifecycle XML's @@ -1076,6 +1082,8 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader case codes.Unavailable, codes.Canceled, codes.DeadlineExceeded: s3a.markOwnerUnreachable(owner) } + } else if upload, ok := multipartPartUpload(r); ok && cond == nil { + createErr = upload.WithFilerClient(false, create) } else if s3a.filerClient != nil { createErr = s3a.withFilerClientFailover(context.Background(), "", false, cond == nil, create) } else { @@ -1184,7 +1192,11 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader // can hide a commit (issue #11366) and the filer can fail after // inserting the entry (issue #11387), so the entry's presence — not // the error class — decides the chunks' fate. - landed, absent := s3a.confirmCreateLanded(filePath, bucket, object, entry, chunkResult.FileChunks, finalize) + routeObject := object + if upload, ok := multipartPartUpload(r); ok { + routeObject = upload.object + } + landed, absent := s3a.confirmCreateLanded(filePath, bucket, object, routeObject, entry, chunkResult.FileChunks, finalize) if landed { createCode = s3err.ErrNone } @@ -1397,16 +1409,17 @@ func chunkNeedleExists(ctx context.Context, fileUrl, jwt string) bool { // finalization the error skipped then runs under the object write lock, and // landed reports success — while absent requires every filer the create could // have committed on to lack the entry, the only outcome where the uploaded -// chunks are orphaned. -func (s3a *S3ApiServer) confirmCreateLanded(filePath, bucket, object string, entry *filer_pb.Entry, uploaded []*filer_pb.FileChunk, finalize *putFinalize) (landed, absent bool) { +// chunks are orphaned. routeObject picks the owner filers to check: the object +// itself, or for a staged part the object its upload completes into. +func (s3a *S3ApiServer) confirmCreateLanded(filePath, bucket, object, routeObject string, entry *filer_pb.Entry, uploaded []*filer_pb.FileChunk, finalize *putFinalize) (landed, absent bool) { dir, name := path.Dir(filePath), path.Base(filePath) - owner := s3a.routableWriteOwner(bucket, object) + owner := s3a.routableWriteOwner(bucket, routeObject) lookupCtx, cancel := context.WithTimeout(context.Background(), createLookupTimeout) defer cancel() // Verify, finalize, and roll back inside one critical section: a concurrent // write to the same key must not slip in between them. s3a.withObjectWriteLock(bucket, object, nil, func() s3err.ErrorCode { - targets := s3a.createTargetFilers(owner, bucket, object) + targets := s3a.createTargetFilers(owner, bucket, routeObject) existing, uncertain := s3a.lookupCommittedEntry(lookupCtx, targets, dir, name) if existing == nil { absent = len(targets) > 0 && !uncertain diff --git a/weed/s3api/s3api_object_routed_upload.go b/weed/s3api/s3api_object_routed_upload.go new file mode 100644 index 0000000000..50075df7de --- /dev/null +++ b/weed/s3api/s3api_object_routed_upload.go @@ -0,0 +1,141 @@ +package s3api + +import ( + "context" + "errors" + "sort" + + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" +) + +// uploadFilerClient reads and writes one multipart upload's .uploads/ +// directory on the destination object's write owner, the filer the upload's +// completion commits on. Gateways on independent filer stores then see an +// upload, and its parts, as soon as the owner stored them. +type uploadFilerClient struct { + s3a *S3ApiServer + bucket string + object string +} + +func (s3a *S3ApiServer) uploadFiler(bucket, object string) uploadFilerClient { + return uploadFilerClient{s3a: s3a, bucket: bucket, object: object} +} + +func (u uploadFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error { + owner := u.s3a.routableWriteOwner(u.bucket, u.object) + if owner == "" || u.s3a.filerClient == nil { + return u.s3a.WithFilerClient(streamingMode, fn) + } + if u.s3a.ownerRecentlyUnreachable(owner) { + owner = "" + } + return u.s3a.withFilerClientFailover(context.Background(), owner, streamingMode, true, fn) +} + +func (u uploadFilerClient) AdjustedUrl(location *filer_pb.Location) string { + return u.s3a.AdjustedUrl(location) +} + +func (u uploadFilerClient) GetDataCenter() string { + return u.s3a.GetDataCenter() +} + +// getEntry reads the upload record. Right after a ring change a part write can +// leave the new owner a bare directory before the record replicated there, so +// the prior owner's record wins over one without the upload's key, and a prior +// owner that cannot answer fails the lookup rather than reading as no upload. +func (u uploadFilerClient) getEntry(parentDirectoryPath, entryName string) (*filer_pb.Entry, error) { + entry, err := u.s3a.getObjectScopedEntryRoutedByKey(u.bucket, u.object, parentDirectoryPath, entryName) + if (err != nil && !errors.Is(err, filer_pb.ErrNotFound)) || isMultipartUploadEntry(entry) { + return entry, err + } + prior := u.s3a.priorWriteOwner(u.bucket, u.object) + if prior == "" { + return entry, err + } + priorEntry, priorErr := u.s3a.lookupEntryOnFiler(context.Background(), prior, parentDirectoryPath, entryName) + if priorErr != nil && !isFilerNotFound(priorErr) { + return nil, priorErr + } + if priorErr == nil && isMultipartUploadEntry(priorEntry) { + return priorEntry, nil + } + return entry, err +} + +// list reads the owner's listing. While a ring change is in its cooling-off +// window the prior owner may still hold parts the new owner has not received, +// so its listing is merged in, the owner's entry winning a name both hold. +func (u uploadFilerClient) list(parentDirectoryPath, prefix, startFrom string, inclusive bool, limit uint32) ([]*filer_pb.Entry, bool, error) { + entries, isLast, err := u.s3a.listObjectScopedEntriesRoutedByKey(u.bucket, u.object, parentDirectoryPath, prefix, startFrom, inclusive, limit) + if err != nil && !errors.Is(err, filer_pb.ErrNotFound) { + return entries, isLast, err + } + prior := u.s3a.priorWriteOwner(u.bucket, u.object) + if prior == "" { + return entries, isLast, err + } + var priorEntries []*filer_pb.Entry + priorLast := false + priorErr := pb.WithFilerClient(true, 0, prior, u.s3a.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + return filer_pb.SeaweedList(context.Background(), client, parentDirectoryPath, prefix, func(entry *filer_pb.Entry, isLastEntry bool) error { + priorEntries = append(priorEntries, entry) + priorLast = priorLast || isLastEntry + return nil + }, startFrom, inclusive, limit) + }) + if priorErr != nil && !isFilerNotFound(priorErr) && len(entries) == 0 { + // with nothing on the owner yet, every part may be on the prior owner + return nil, false, priorErr + } + if priorErr != nil || len(priorEntries) == 0 { + return entries, isLast, err + } + merged, mergedLast := mergeEntryListings(entries, isLast, priorEntries, priorLast, limit) + return merged, mergedLast, nil +} + +// mergeEntryListings merges two pages of one directory listed from the same +// cursor, taking primary's entry for a name both hold. Each page holds the +// first limit names past the cursor, so the merged first limit are exact. +func mergeEntryListings(primary []*filer_pb.Entry, primaryLast bool, secondary []*filer_pb.Entry, secondaryLast bool, limit uint32) ([]*filer_pb.Entry, bool) { + byName := make(map[string]*filer_pb.Entry, len(primary)+len(secondary)) + for _, entry := range secondary { + byName[entry.Name] = entry + } + for _, entry := range primary { + byName[entry.Name] = entry + } + merged := make([]*filer_pb.Entry, 0, len(byName)) + for _, entry := range byName { + merged = append(merged, entry) + } + sort.Slice(merged, func(i, j int) bool { return merged[i].Name < merged[j].Name }) + isLast := primaryLast && secondaryLast + if limit > 0 && uint32(len(merged)) > limit { + merged, isLast = merged[:limit], false + } + return merged, isLast +} + +func (u uploadFilerClient) exists(parentDirectoryPath, entryName string, isDirectory bool) (bool, error) { + return filer_pb.Exists(context.Background(), u, parentDirectoryPath, entryName, isDirectory) +} + +func (u uploadFilerClient) mkdir(parentDirectoryPath, dirName string, fn func(entry *filer_pb.Entry)) error { + return filer_pb.Mkdir(context.Background(), u, parentDirectoryPath, dirName, fn) +} + +func (u uploadFilerClient) mkFile(parentDirectoryPath, fileName string, chunks []*filer_pb.FileChunk, fn func(entry *filer_pb.Entry)) error { + return filer_pb.MkFile(context.Background(), u, parentDirectoryPath, fileName, chunks, fn) +} + +func (u uploadFilerClient) rm(ctx context.Context, parentDirectoryPath, entryName string, isDeleteData, isRecursive bool) error { + return retryFilerOp(ctx, "rm "+parentDirectoryPath+"/"+entryName, func() error { + return u.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + return doDeleteEntry(ctx, client, parentDirectoryPath, entryName, isDeleteData, isRecursive) + }) + }) +} diff --git a/weed/s3api/s3api_object_routed_upload_test.go b/weed/s3api/s3api_object_routed_upload_test.go new file mode 100644 index 0000000000..5d8f0ec115 --- /dev/null +++ b/weed/s3api/s3api_object_routed_upload_test.go @@ -0,0 +1,350 @@ +package s3api + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/cluster" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// The gateway's own filer never receives the owner's metadata here, so every +// multipart step after CreateMultipartUpload must find the upload and its +// parts on the object's write owner. +func TestMultipartUploadOnOwnerFiler(t *testing.T) { + util_http.InitGlobalHttpClient() + volume := startFakeVolumeServer(t) + owner := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}} + replica := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}} + ownerAddr := startFakeFiler(t, owner) + replicaAddr := startFakeFiler(t, replica) + s3a := newPutTestServer(t, replicaAddr) + s3a.iam = &IdentityAccessManagement{} + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"}) + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, ownerAddr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{ownerAddr}, 1) + + has := func(f *ambiguousPutFiler, path string) bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.entries[path] != nil + } + partsUnder := func(f *ambiguousPutFiler, dir string) int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for key, entry := range f.entries { + if strings.HasPrefix(key, dir+"/") && !entry.IsDirectory { + n++ + } + } + return n + } + initiate := func() string { + r := httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil) + out, code := s3a.createMultipartUpload(r, &s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")}) + require.Equal(t, s3err.ErrNone, code) + uploadDir := "/buckets/b/.uploads/" + *out.UploadId + require.True(t, has(owner, uploadDir), "the upload must be created on the owner") + require.False(t, has(replica, uploadDir), "the gateway's filer must still be behind") + return *out.UploadId + } + uploadPart := func(uploadID string, partNumber int, body string) string { + r := httptest.NewRequest(http.MethodPut, fmt.Sprintf("/b/dir/object?partNumber=%d&uploadId=%s", partNumber, uploadID), strings.NewReader(body)) + r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"}) + w := httptest.NewRecorder() + s3a.PutObjectPartHandler(w, r) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + etag := w.Header()["ETag"] + require.Len(t, etag, 1) + return etag[0] + } + + uploadID := initiate() + uploadDir := "/buckets/b/.uploads/" + uploadID + etag1 := uploadPart(uploadID, 1, strings.Repeat("a", 5*1024*1024)) + etag2 := uploadPart(uploadID, 2, "tail") + require.Equal(t, 2, partsUnder(owner, uploadDir), "parts must be stored next to the upload") + require.Equal(t, 0, partsUnder(replica, uploadDir)) + + listed, code := s3a.listObjectParts(&s3.ListPartsInput{ + Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: aws.String(uploadID), + MaxParts: aws.Int64(1000), PartNumberMarker: aws.Int64(0), + }) + require.Equal(t, s3err.ErrNone, code) + require.Len(t, listed.Part, 2) + + complete := httptest.NewRequest(http.MethodPost, "/b/dir/object?uploadId="+uploadID, nil) + _, code = s3a.completeMultipartUpload(complete, &s3.CompleteMultipartUploadInput{ + Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: aws.String(uploadID), + }, &CompleteMultipartUpload{Parts: []CompletedPart{{ETag: etag1, PartNumber: 1}, {ETag: etag2, PartNumber: 2}}}) + require.Equal(t, s3err.ErrNone, code) + require.True(t, has(owner, "/buckets/b/dir/object"), "the object must commit on the owner") + require.False(t, has(owner, uploadDir), "completion must remove the upload") + + aborted := initiate() + uploadPart(aborted, 1, "data") + _, code = s3a.abortMultipartUpload(&s3.AbortMultipartUploadInput{ + Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: aws.String(aborted), + }) + require.Equal(t, s3err.ErrNone, code) + require.False(t, has(owner, "/buckets/b/.uploads/"+aborted), "abort must remove the upload") +} + +// A part create the owner stored but answered with an error must be found +// there before its chunks are freed, even though the owner is not one of the +// gateway's own filers. +func TestMultipartPartCreateErrorChecksUploadOwner(t *testing.T) { + util_http.InitGlobalHttpClient() + volume := startFakeVolumeServer(t) + owner := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}, + respError: "create parent directories: i/o timeout"} + replica := &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}} + ownerAddr := startFakeFiler(t, owner) + s3a := newPutTestServer(t, startFakeFiler(t, replica)) + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, ownerAddr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{ownerAddr}, 1) + + r := withMultipartPart(httptest.NewRequest(http.MethodPut, "/b/dir/object?partNumber=1&uploadId=up1", nil), s3a.uploadFiler("b", "dir/object")) + etag, code, _ := s3a.putToFiler(r, "/buckets/b/.uploads/up1/0001_x.part", strings.NewReader("part data"), "b", "", 1, 0, nil, false, "/buckets/b/dir/object") + require.Equal(t, s3err.ErrNone, code) + require.NotEmpty(t, etag) + require.Empty(t, volume.deleted(), "chunks under the stored part must survive") +} + +// Right after the key moves to a new owner that has not received the upload +// yet, ListParts finds the record and its parts on the prior owner. +func TestListPartsAfterOwnerChange(t *testing.T) { + util_http.InitGlobalHttpClient() + volume := startFakeVolumeServer(t) + prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}} + next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}} + priorAddr := startFakeFiler(t, prior) + nextAddr := startFakeFiler(t, next) + s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}})) + s3a.iam = &IdentityAccessManagement{} + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"}) + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1) + + out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil), + &s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")}) + require.Equal(t, s3err.ErrNone, code) + r := httptest.NewRequest(http.MethodPut, "/b/dir/object?partNumber=1&uploadId="+*out.UploadId, strings.NewReader("data")) + r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"}) + w := httptest.NewRecorder() + s3a.PutObjectPartHandler(w, r) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2) + require.Equal(t, priorAddr, s3a.priorWriteOwner("b", "dir/object")) + + listed, code := s3a.listObjectParts(&s3.ListPartsInput{ + Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId, + MaxParts: aws.Int64(1000), PartNumberMarker: aws.Int64(0), + }) + require.Equal(t, s3err.ErrNone, code) + require.Len(t, listed.Part, 1) +} + +// A part routed to a new owner that has not received the upload yet makes that +// filer create a bare .uploads/ directory; the upload must stay open. +func TestUploadPartAfterOwnerChange(t *testing.T) { + util_http.InitGlobalHttpClient() + volume := startFakeVolumeServer(t) + prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}} + next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}} + priorAddr := startFakeFiler(t, prior) + nextAddr := startFakeFiler(t, next) + s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}})) + s3a.iam = &IdentityAccessManagement{} + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"}) + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1) + + out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil), + &s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")}) + require.Equal(t, s3err.ErrNone, code) + uploadDir := "/buckets/b/.uploads/" + *out.UploadId + next.beforeCreate = func() { + if next.entries[uploadDir] == nil { + next.entries[uploadDir] = &filer_pb.Entry{Name: *out.UploadId, IsDirectory: true} + } + } + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2) + + r := httptest.NewRequest(http.MethodPut, "/b/dir/object?partNumber=1&uploadId="+*out.UploadId, strings.NewReader("data")) + r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"}) + w := httptest.NewRecorder() + s3a.PutObjectPartHandler(w, r) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + + listed, code := s3a.listObjectParts(&s3.ListPartsInput{ + Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId, + MaxParts: aws.Int64(1000), PartNumberMarker: aws.Int64(0), + }) + require.Equal(t, s3err.ErrNone, code) + require.Len(t, listed.Part, 1) +} + +// A new owner that received only some of the parts so far lists them together +// with the rest still on the prior owner. +func TestListPartsMergesPriorOwnerParts(t *testing.T) { + util_http.InitGlobalHttpClient() + volume := startFakeVolumeServer(t) + prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}} + next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}} + priorAddr := startFakeFiler(t, prior) + nextAddr := startFakeFiler(t, next) + s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}})) + s3a.iam = &IdentityAccessManagement{} + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"}) + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1) + + out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil), + &s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")}) + require.Equal(t, s3err.ErrNone, code) + uploadDir := "/buckets/b/.uploads/" + *out.UploadId + etags := map[int]string{} + for partNumber, body := range []string{strings.Repeat("a", 5*1024*1024), "tail"} { + r := httptest.NewRequest(http.MethodPut, fmt.Sprintf("/b/dir/object?partNumber=%d&uploadId=%s", partNumber+1, *out.UploadId), strings.NewReader(body)) + r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"}) + w := httptest.NewRecorder() + s3a.PutObjectPartHandler(w, r) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + require.Len(t, w.Header()["ETag"], 1) + etags[partNumber+1] = w.Header()["ETag"][0] + } + + prior.mu.Lock() + for key, entry := range prior.entries { + if key == uploadDir || strings.HasPrefix(key, uploadDir+"/0001") { + next.entries[key] = entry + } + } + prior.mu.Unlock() + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2) + + listParts := func(maxParts, marker int64) *ListPartsResult { + listed, code := s3a.listObjectParts(&s3.ListPartsInput{ + Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId, + MaxParts: aws.Int64(maxParts), PartNumberMarker: aws.Int64(marker), + }) + require.Equal(t, s3err.ErrNone, code) + return listed + } + require.Len(t, listParts(1000, 0).Part, 2) + first := listParts(1, 0) + require.Len(t, first.Part, 1) + require.Equal(t, int64(1), *first.Part[0].PartNumber) + require.True(t, *first.IsTruncated) + second := listParts(1, 1) + require.Len(t, second.Part, 1) + require.Equal(t, int64(2), *second.Part[0].PartNumber) + require.False(t, *second.IsTruncated) + + _, code = s3a.completeMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploadId="+*out.UploadId, nil), + &s3.CompleteMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId}, + &CompleteMultipartUpload{Parts: []CompletedPart{{ETag: etags[1], PartNumber: 1}, {ETag: etags[2], PartNumber: 2}}}) + require.Equal(t, s3err.ErrNone, code) +} + +type failingListFiler struct { + *ambiguousPutFiler +} + +func (f failingListFiler) ListEntries(*filer_pb.ListEntriesRequest, filer_pb.SeaweedFiler_ListEntriesServer) error { + return status.Error(codes.Unavailable, "store unavailable") +} + +// With no parts on the new owner yet, a prior owner that cannot be listed may +// hold all of them, so completion must not read the upload as empty. +func TestCompleteFailsWhenPriorOwnerListingFails(t *testing.T) { + util_http.InitGlobalHttpClient() + volume := startFakeVolumeServer(t) + prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}} + next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}} + priorAddr := startFakeFiler(t, failingListFiler{prior}) + nextAddr := startFakeFiler(t, next) + s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}})) + s3a.iam = &IdentityAccessManagement{} + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"}) + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1) + + out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil), + &s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")}) + require.Equal(t, s3err.ErrNone, code) + r := httptest.NewRequest(http.MethodPut, "/b/dir/object?partNumber=1&uploadId="+*out.UploadId, strings.NewReader("data")) + r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"}) + w := httptest.NewRecorder() + s3a.PutObjectPartHandler(w, r) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + require.Len(t, w.Header()["ETag"], 1) + + uploadDir := "/buckets/b/.uploads/" + *out.UploadId + prior.mu.Lock() + next.entries[uploadDir] = prior.entries[uploadDir] + prior.mu.Unlock() + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2) + + _, code = s3a.completeMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploadId="+*out.UploadId, nil), + &s3.CompleteMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId}, + &CompleteMultipartUpload{Parts: []CompletedPart{{ETag: w.Header()["ETag"][0], PartNumber: 1}}}) + require.Equal(t, s3err.ErrInternalError, code) +} + +// A prior owner that cannot be asked for the upload record must not make an +// upload the new owner has not received yet read as gone. +func TestUploadLookupFailsWhenPriorOwnerUnreachable(t *testing.T) { + util_http.InitGlobalHttpClient() + volume := startFakeVolumeServer(t) + prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}} + next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}} + priorAddr := startFakeFiler(t, prior) + nextAddr := startFakeFiler(t, next) + s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}})) + s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr) + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1) + + out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil), + &s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")}) + require.Equal(t, s3err.ErrNone, code) + uploadDir := "/buckets/b/.uploads/" + *out.UploadId + prior.mu.Lock() + prior.lookupErr, prior.lookupFailKey = status.Error(codes.Unavailable, "store unavailable"), uploadDir + prior.mu.Unlock() + s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2) + + listParts := func() s3err.ErrorCode { + _, code := s3a.listObjectParts(&s3.ListPartsInput{ + Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId, + MaxParts: aws.Int64(1000), PartNumberMarker: aws.Int64(0), + }) + return code + } + require.Equal(t, s3err.ErrInternalError, listParts(), "new owner has nothing yet") + next.mu.Lock() + next.entries[uploadDir] = &filer_pb.Entry{Name: *out.UploadId, IsDirectory: true} + next.mu.Unlock() + require.Equal(t, s3err.ErrInternalError, listParts(), "new owner has only a bare directory") +} diff --git a/weed/s3api/s3api_object_routed_write.go b/weed/s3api/s3api_object_routed_write.go index c90e313dcc..007f2b1067 100644 --- a/weed/s3api/s3api_object_routed_write.go +++ b/weed/s3api/s3api_object_routed_write.go @@ -323,17 +323,17 @@ type uploadRemovalTxn struct { // upload's directory inside the object's commit, after freeing the part // entries the object does not reference. It is nil when the write is not // routed and the upload directory still needs post-commit cleanup. -func (s3a *S3ApiServer) routedUploadRemoval(ctx context.Context, owner pb.ServerAddress, uploadDirectory, bucket, uploadId string, completionState *multipartCompletionState) (*uploadRemovalTxn, error) { +func (s3a *S3ApiServer) routedUploadRemoval(ctx context.Context, owner pb.ServerAddress, uploadDirectory, bucket, object, uploadId string, completionState *multipartCompletionState) (*uploadRemovalTxn, error) { if owner == "" { return nil, nil } // Older filers ignore the mutation's upload-exists condition, so during an // upgrade this gateway-side check keeps the same guard window the // pre-transaction code had. - if exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadId, true); err == nil && !exists { + if _, err := s3a.lookupEntryPreferringOwner(owner, s3a.genUploadsFolder(bucket), uploadId); errors.Is(err, filer_pb.ErrNotFound) { return nil, errUploadRemoved } - if err := s3a.deleteUnusedPartEntries(ctx, uploadDirectory, bucket, uploadId, completionState); err != nil { + if err := s3a.deleteUnusedPartEntries(ctx, uploadDirectory, bucket, object, uploadId, completionState); err != nil { return nil, err } conditionKey, condition := uploadExistsCondition(uploadDirectory) diff --git a/weed/s3api/s3api_object_routed_write_test.go b/weed/s3api/s3api_object_routed_write_test.go index e2ea779f8b..ac1649495c 100644 --- a/weed/s3api/s3api_object_routed_write_test.go +++ b/weed/s3api/s3api_object_routed_write_test.go @@ -649,10 +649,10 @@ func TestWriteMultipartObjectRemovesUploadDir(t *testing.T) { }, } - if removal, err := s3a.routedUploadRemoval(context.Background(), "", "/buckets/b/.uploads/up1", "b", "up1", &multipartCompletionState{}); removal != nil || err != nil { + if removal, err := s3a.routedUploadRemoval(context.Background(), "", "/buckets/b/.uploads/up1", "b", "o", "up1", &multipartCompletionState{}); removal != nil || err != nil { t.Fatalf("unrouted write should not carry the removal, got %+v, %v", removal, err) } - removal, err := s3a.routedUploadRemoval(context.Background(), owner, "/buckets/b/.uploads/up1", "b", "up1", &multipartCompletionState{}) + removal, err := s3a.routedUploadRemoval(context.Background(), owner, "/buckets/b/.uploads/up1", "b", "o", "up1", &multipartCompletionState{}) if err != nil { t.Fatalf("routedUploadRemoval: %v", err) } diff --git a/weed/s3api/s3api_object_versioned_finalize.go b/weed/s3api/s3api_object_versioned_finalize.go index 99b54a2721..0f8bf98367 100644 --- a/weed/s3api/s3api_object_versioned_finalize.go +++ b/weed/s3api/s3api_object_versioned_finalize.go @@ -165,7 +165,7 @@ func (s3a *S3ApiServer) routedMultipartFinalize(owner pb.ServerAddress, bucket, // A filer older than the mutation-condition support silently ignores // putCondition, so on upgrade the guard degrades to this gateway-side // check — the same window the pre-transaction code had. - if exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true); err == nil && !exists { + if exists, err := s3a.uploadFiler(bucket, object).exists(s3a.genUploadsFolder(bucket), uploadID, true); err == nil && !exists { return s3err.ErrNoSuchUpload } putConditionKey, putCondition := uploadExistsCondition(s3a.genUploadsFolder(bucket) + "/" + uploadID) diff --git a/weed/s3api/s3api_storage_class_validation_test.go b/weed/s3api/s3api_storage_class_validation_test.go index de1e8fd343..e545a3f2c6 100644 --- a/weed/s3api/s3api_storage_class_validation_test.go +++ b/weed/s3api/s3api_storage_class_validation_test.go @@ -51,7 +51,7 @@ func TestPutToFilerStorageClass(t *testing.T) { } filePath, object, dest := "/buckets/b/o", "o", "" if tt.part { - r = withMultipartPart(r) + r = withMultipartPart(r, s3a.uploadFiler("b", "o")) filePath, object, dest = "/buckets/b/.uploads/u/0001.part", "", "/buckets/b/o" } _, code, _ := s3a.putToFiler(r, filePath, strings.NewReader("hello world"), "b", object, 1, 0, nil, false, dest)