seaweedfs/test/s3/sse/s3_sse_uploadpartcopy_integration_test.go

package sse_test

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"strings"
	"testing"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/aws/aws-sdk-go-v2/service/s3/types"
	"github.com/stretchr/testify/require"
)

// TestSSES3MultipartUploadPartCopyIntegration pins the fix for issue #8908.
//
// Docker Registry's S3 storage driver finalizes blob uploads via a server-side
// "Move" pattern: a streaming PUT/multipart upload to a temporary key, then
// CreateMultipartUpload + UploadPartCopy(s) + CompleteMultipartUpload to put
// the bytes at the final blob path. Under bucket-default SSE-S3, every push
// goes through this UploadPartCopy step.
//
// Before the fix, copyChunksForRange did a raw byte copy that left the
// destination's part chunks SseType=NONE. Then completedMultipartChunk
// (PR #9224) saw NONE chunks in an SSE-S3 multipart upload and "backfilled"
// SSE-S3 metadata with IVs derived from the destination upload's baseIV. But
// the bytes on disk had been encrypted with the SOURCE upload's key+baseIV,
// so the read path decrypted with the wrong IV — yielding deterministic byte
// corruption on GET (the "Digest did not match" symptom kubelet surfaces).
//
// This test reproduces the exact shape: a 39MB plaintext source (single
// PutObject — auto-chunked into multiple internal SSE-S3 chunks on disk
// because of bucket-default SSE-S3), then a fresh multipart upload at a new
// destination key with two UploadPartCopy parts (32MB + 7MB) and Complete.
// The full GET must SHA back to what was uploaded.
//
// The function name contains both "Multipart" and "Integration" so it is matched
// by the `.*Multipart.*Integration` pattern in .github/workflows/s3-sse-tests.yml
// and the `TestSSE.*Integration` pattern in test/s3/sse/Makefile, ensuring this
// regression coverage runs in CI.
func TestSSES3MultipartUploadPartCopyIntegration(t *testing.T) {
	ctx := context.Background()
	client, err := createS3Client(ctx, defaultConfig)
	require.NoError(t, err, "Failed to create S3 client")

	bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"sse-s3-uploadpartcopy-")
	require.NoError(t, err, "Failed to create test bucket")
	defer cleanupTestBucket(ctx, client, bucketName)

	// Bucket-default SSE-S3 — same setup Docker Registry uses.
	_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
		Bucket: aws.String(bucketName),
		ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
			Rules: []types.ServerSideEncryptionRule{
				{
					ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
						SSEAlgorithm: types.ServerSideEncryptionAes256,
					},
				},
			},
		},
	})
	require.NoError(t, err, "Failed to set bucket default SSE-S3")

	// Source: 39MB single PutObject (auto-chunked internally into 5 SSE-S3 chunks).
	const sourceSize = 39 * 1024 * 1024
	sourceData := generateTestData(sourceSize)
	expectedSHA := sha256.Sum256(sourceData)

	_, err = client.PutObject(ctx, &s3.PutObjectInput{
		Bucket: aws.String(bucketName),
		Key:    aws.String("source-blob"),
		Body:   bytes.NewReader(sourceData),
	})
	require.NoError(t, err, "Failed to upload source object")

	// Sanity check: the source itself must round-trip correctly.
	{
		resp, err := client.GetObject(ctx, &s3.GetObjectInput{
			Bucket: aws.String(bucketName),
			Key:    aws.String("source-blob"),
		})
		require.NoError(t, err, "Failed to GET source")
		got, err := io.ReadAll(resp.Body)
		resp.Body.Close()
		require.NoError(t, err)
		require.Equal(t, expectedSHA, sha256.Sum256(got), "source object must round-trip")
	}

	cases := []struct {
		name string
		// part definitions: each entry is (start, end) byte range to copy.
		parts [][2]int64
	}{
		{
			// Docker Registry's typical Move shape for blobs around 40MB:
			// one 32MB part + one tail part. This is exactly the metadata
			// shape the user reported (5 dst chunks across 2 multipart parts).
			name: "DockerRegistry_32MB_Plus_Tail",
			parts: [][2]int64{
				{0, 32*1024*1024 - 1},
				{32 * 1024 * 1024, sourceSize - 1},
			},
		},
		{
			// Single full-object UploadPartCopy.
			name: "Single_Full_Object_Copy",
			parts: [][2]int64{
				{0, sourceSize - 1},
			},
		},
		{
			// Many small range-copies — exercises the per-part-local-offset
			// IV math under varied chunk-overlap shapes.
			name: "Many_5MB_Ranges",
			parts: [][2]int64{
				{0, 5*1024*1024 - 1},
				{5 * 1024 * 1024, 10*1024*1024 - 1},
				{10 * 1024 * 1024, 15*1024*1024 - 1},
				{15 * 1024 * 1024, 20*1024*1024 - 1},
				{20 * 1024 * 1024, sourceSize - 1},
			},
		},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			dstKey := "dest-" + strings.ToLower(strings.ReplaceAll(tc.name, "_", "-"))

			createResp, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
				Bucket: aws.String(bucketName),
				Key:    aws.String(dstKey),
			})
			require.NoError(t, err, "CreateMultipartUpload")
			uploadID := aws.ToString(createResp.UploadId)

			completedParts := make([]types.CompletedPart, 0, len(tc.parts))
			for i, rng := range tc.parts {
				partNumber := int32(i + 1)
				resp, err := client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
					Bucket:          aws.String(bucketName),
					Key:             aws.String(dstKey),
					PartNumber:      aws.Int32(partNumber),
					UploadId:        aws.String(uploadID),
					CopySource:      aws.String(bucketName + "/source-blob"),
					CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", rng[0], rng[1])),
				})
				require.NoErrorf(t, err, "UploadPartCopy part %d range=[%d,%d]", partNumber, rng[0], rng[1])
				completedParts = append(completedParts, types.CompletedPart{
					ETag:       resp.CopyPartResult.ETag,
					PartNumber: aws.Int32(partNumber),
				})
			}

			_, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
				Bucket:          aws.String(bucketName),
				Key:             aws.String(dstKey),
				UploadId:        aws.String(uploadID),
				MultipartUpload: &types.CompletedMultipartUpload{Parts: completedParts},
			})
			require.NoError(t, err, "CompleteMultipartUpload")

			// Two-pass verification: full GET (Docker Registry / Kubelet shape).
			resp, err := client.GetObject(ctx, &s3.GetObjectInput{
				Bucket: aws.String(bucketName),
				Key:    aws.String(dstKey),
			})
			require.NoError(t, err, "GetObject")
			defer resp.Body.Close()

			h := sha256.New()
			n, err := io.Copy(h, resp.Body)
			require.NoError(t, err, "stream GET body")
			require.Equal(t, int64(sourceSize), n, "GET length")

			var actual [32]byte
			copy(actual[:], h.Sum(nil))

			require.Equalf(t, expectedSHA, actual,
				"UploadPartCopy SHA mismatch (#8908):\n  expected sha256:%s\n  got      sha256:%s\n  parts=%d  size=%d",
				hex.EncodeToString(expectedSHA[:]),
				hex.EncodeToString(actual[:]),
				len(tc.parts), n)
		})
	}
}