From 17ad5a1419fba94ddd285544d01fd8d85ac3b5f4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 19 Sep 2026 23:28:16 -0700 Subject: [PATCH] s3: accept FULL_OBJECT checksums without per-part checksums at complete COMPOSITE uploads must still carry every part checksum in the complete request, but FULL_OBJECT uploads may instead supply the whole-object checksum in an x-amz-checksum-* request header. Compare that header against the computed object checksum and return BadDigest on mismatch, matching AWS. --- test/s3/checksum/issue11401_repro_test.go | 55 +++++++++++++++++++++++ weed/s3api/filer_multipart.go | 16 +++++-- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/test/s3/checksum/issue11401_repro_test.go b/test/s3/checksum/issue11401_repro_test.go index 00c6ce379..79f0a8742 100644 --- a/test/s3/checksum/issue11401_repro_test.go +++ b/test/s3/checksum/issue11401_repro_test.go @@ -212,6 +212,61 @@ func TestCompleteMultipartUploadValidatesPartChecksums(t *testing.T) { require.NoError(t, complete(part.ChecksumSHA256)) } +// FULL_OBJECT uploads need no per-part checksums in the complete request; the +// whole-object checksum travels in a request header and is validated against +// the computed value (BadDigest on mismatch), as on AWS. +func TestCompleteMultipartUploadFullObjectChecksum(t *testing.T) { + client := newWhenRequiredChecksumClient(t) + + bucket := uniqueBucket() + createBucket(t, client, bucket) + defer cleanupBucket(t, client, bucket) + + body := bytes.Repeat([]byte("w"), 1024) + key := "full-object" + + create, err := client.CreateMultipartUpload(context.Background(), &s3.CreateMultipartUploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + ChecksumAlgorithm: types.ChecksumAlgorithmCrc64nvme, + }) + require.NoError(t, err) + require.Equal(t, types.ChecksumTypeFullObject, create.ChecksumType) + + part, err := client.UploadPart(context.Background(), &s3.UploadPartInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + UploadId: create.UploadId, + PartNumber: aws.Int32(1), + Body: bytes.NewReader(body), + }) + require.NoError(t, err) + require.NotEmpty(t, aws.ToString(part.ChecksumCRC64NVME)) + + complete := func(objectChecksum *string) error { + _, err := client.CompleteMultipartUpload(context.Background(), &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + UploadId: create.UploadId, + ChecksumCRC64NVME: objectChecksum, + MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{{ + ETag: part.ETag, + PartNumber: aws.Int32(1), + }}}, + }) + return err + } + + var apiErr smithy.APIError + wrong := base64.StdEncoding.EncodeToString(func() []byte { s := sha256.Sum256(body); return s[:] }()) + err = complete(aws.String(wrong)) + require.Error(t, err) + require.True(t, errors.As(err, &apiErr)) + require.Equal(t, "BadDigest", apiErr.ErrorCode()) + + require.NoError(t, complete(part.ChecksumCRC64NVME)) +} + // UploadPart with a checksum algorithm that conflicts with the one declared at // CreateMultipartUpload is rejected, as on AWS. func TestMultipartPartConflictingAlgorithm(t *testing.T) { diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index f9fdc763f..e5dd9c8c1 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -591,15 +591,22 @@ func (s3a *S3ApiServer) prepareMultipartCompletionState(r *http.Request, input * provided[part.PartNumber] = part.GetChecksum(checksumHeaderName) } for _, partNumber := range completedPartNumbers { - if provided[partNumber] == "" { - return nil, nil, s3err.ErrInvalidRequest + providedChecksum := provided[partNumber] + if providedChecksum == "" { + // COMPOSITE uploads must carry every part checksum in the + // request; FULL_OBJECT uploads instead carry the whole-object + // checksum in a request header. + if checksumType == s3_constants.ChecksumTypeComposite { + return nil, nil, s3err.ErrInvalidRequest + } + continue } raw, _, decodeErr := decodePartChecksum(partNumber, partEntries[partNumber], checksumHeaderName) if decodeErr != nil { glog.Errorf("completeMultipartUpload: %v", decodeErr) return nil, nil, s3err.ErrInvalidPart } - if provided[partNumber] != base64.StdEncoding.EncodeToString(raw) { + if providedChecksum != base64.StdEncoding.EncodeToString(raw) { return nil, nil, s3err.ErrBadDigest } } @@ -614,6 +621,9 @@ func (s3a *S3ApiServer) prepareMultipartCompletionState(r *http.Request, input * glog.Errorf("completeMultipartUpload: %s checksum computation failed: %v", checksumType, checksumErr) return nil, nil, s3err.ErrInvalidPart } + if objectChecksum := r.Header.Get(checksumHeaderName); objectChecksum != "" && objectChecksum != checksumValue { + return nil, nil, s3err.ErrBadDigest + } } // Fold after the boundaries captured byte offsets; finalParts is flat