mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-08 15:41:15 +02:00
s3: return the multipart object checksum in the CompleteMultipartUpload response (#11101)
* s3: return the multipart object checksum in the CompleteMultipartUpload body S3 carries the flexible-checksum members of CompleteMultipartUploadResult in the XML body, not in response headers, so every SDK read back an empty checksum from an upload that asked for one. Claude-Session: https://claude.ai/code/session_01Huux1uh7JxAbf8yypMYrMk * s3: echo the checksum algorithm and type from CreateMultipartUpload The upload directory already records both, but the response dropped them, so a client could not confirm which checksum its parts had to carry. Claude-Session: https://claude.ai/code/session_01Huux1uh7JxAbf8yypMYrMk * test: multipart upload reports the object checksum it was asked for Covers every algorithm end to end: the create response echoes the algorithm and type, the complete response carries the checksum, and it matches what a later HEAD reports. Claude-Session: https://claude.ai/code/session_01Huux1uh7JxAbf8yypMYrMk
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"net/http/httputil"
|
"net/http/httputil"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/aws/aws-sdk-go-v2/aws"
|
"github.com/aws/aws-sdk-go-v2/aws"
|
||||||
@@ -17,15 +18,18 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMultipartCopyPreservesChecksum(t *testing.T) {
|
// newTrailerChecksumClient returns a client that can send flexible checksums.
|
||||||
// aws-sdk-go-v2 sends flexible checksums as unsigned streaming trailers, which
|
// aws-sdk-go-v2 sends them as unsigned streaming trailers, which it refuses over
|
||||||
// it refuses over plain HTTP, so front the HTTP endpoint with a TLS proxy.
|
// plain HTTP, so front the HTTP endpoint with a TLS proxy.
|
||||||
|
func newTrailerChecksumClient(t *testing.T) *s3.Client {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
target, err := url.Parse(defaultConfig.Endpoint)
|
target, err := url.Parse(defaultConfig.Endpoint)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
proxy := httputil.NewSingleHostReverseProxy(target)
|
proxy := httputil.NewSingleHostReverseProxy(target)
|
||||||
server := httptest.NewTLSServer(proxy)
|
server := httptest.NewTLSServer(proxy)
|
||||||
defer server.Close()
|
t.Cleanup(server.Close)
|
||||||
|
|
||||||
cfg, err := config.LoadDefaultConfig(context.Background(),
|
cfg, err := config.LoadDefaultConfig(context.Background(),
|
||||||
config.WithRegion(defaultConfig.Region),
|
config.WithRegion(defaultConfig.Region),
|
||||||
@@ -34,10 +38,14 @@ func TestMultipartCopyPreservesChecksum(t *testing.T) {
|
|||||||
config.WithHTTPClient(server.Client()),
|
config.WithHTTPClient(server.Client()),
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
|
return s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||||
o.BaseEndpoint = aws.String(server.URL)
|
o.BaseEndpoint = aws.String(server.URL)
|
||||||
o.UsePathStyle = true
|
o.UsePathStyle = true
|
||||||
})
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartCopyPreservesChecksum(t *testing.T) {
|
||||||
|
client := newTrailerChecksumClient(t)
|
||||||
|
|
||||||
bucket := uniqueBucket()
|
bucket := uniqueBucket()
|
||||||
createBucket(t, client, bucket)
|
createBucket(t, client, bucket)
|
||||||
@@ -122,3 +130,120 @@ func TestMultipartCopyPreservesChecksum(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A multipart upload that asked for a checksum must report it in the
|
||||||
|
// CompleteMultipartUpload response, alongside the algorithm and type that
|
||||||
|
// CreateMultipartUpload echoed back.
|
||||||
|
func TestMultipartUploadReturnsObjectChecksum(t *testing.T) {
|
||||||
|
client := newTrailerChecksumClient(t)
|
||||||
|
|
||||||
|
bucket := uniqueBucket()
|
||||||
|
createBucket(t, client, bucket)
|
||||||
|
defer cleanupBucket(t, client, bucket)
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
algorithm types.ChecksumAlgorithm
|
||||||
|
expectedType types.ChecksumType
|
||||||
|
completeSum func(*s3.CompleteMultipartUploadOutput) *string
|
||||||
|
headSum func(*s3.HeadObjectOutput) *string
|
||||||
|
partSum func(*s3.UploadPartOutput) *string
|
||||||
|
setPart func(*types.CompletedPart, *string)
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
algorithm: types.ChecksumAlgorithmCrc32,
|
||||||
|
expectedType: types.ChecksumTypeComposite,
|
||||||
|
completeSum: func(o *s3.CompleteMultipartUploadOutput) *string { return o.ChecksumCRC32 },
|
||||||
|
headSum: func(o *s3.HeadObjectOutput) *string { return o.ChecksumCRC32 },
|
||||||
|
partSum: func(o *s3.UploadPartOutput) *string { return o.ChecksumCRC32 },
|
||||||
|
setPart: func(p *types.CompletedPart, v *string) { p.ChecksumCRC32 = v },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
algorithm: types.ChecksumAlgorithmCrc32c,
|
||||||
|
expectedType: types.ChecksumTypeComposite,
|
||||||
|
completeSum: func(o *s3.CompleteMultipartUploadOutput) *string { return o.ChecksumCRC32C },
|
||||||
|
headSum: func(o *s3.HeadObjectOutput) *string { return o.ChecksumCRC32C },
|
||||||
|
partSum: func(o *s3.UploadPartOutput) *string { return o.ChecksumCRC32C },
|
||||||
|
setPart: func(p *types.CompletedPart, v *string) { p.ChecksumCRC32C = v },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
algorithm: types.ChecksumAlgorithmCrc64nvme,
|
||||||
|
expectedType: types.ChecksumTypeFullObject,
|
||||||
|
completeSum: func(o *s3.CompleteMultipartUploadOutput) *string { return o.ChecksumCRC64NVME },
|
||||||
|
headSum: func(o *s3.HeadObjectOutput) *string { return o.ChecksumCRC64NVME },
|
||||||
|
partSum: func(o *s3.UploadPartOutput) *string { return o.ChecksumCRC64NVME },
|
||||||
|
setPart: func(p *types.CompletedPart, v *string) { p.ChecksumCRC64NVME = v },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
algorithm: types.ChecksumAlgorithmSha1,
|
||||||
|
expectedType: types.ChecksumTypeComposite,
|
||||||
|
completeSum: func(o *s3.CompleteMultipartUploadOutput) *string { return o.ChecksumSHA1 },
|
||||||
|
headSum: func(o *s3.HeadObjectOutput) *string { return o.ChecksumSHA1 },
|
||||||
|
partSum: func(o *s3.UploadPartOutput) *string { return o.ChecksumSHA1 },
|
||||||
|
setPart: func(p *types.CompletedPart, v *string) { p.ChecksumSHA1 = v },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
algorithm: types.ChecksumAlgorithmSha256,
|
||||||
|
expectedType: types.ChecksumTypeComposite,
|
||||||
|
completeSum: func(o *s3.CompleteMultipartUploadOutput) *string { return o.ChecksumSHA256 },
|
||||||
|
headSum: func(o *s3.HeadObjectOutput) *string { return o.ChecksumSHA256 },
|
||||||
|
partSum: func(o *s3.UploadPartOutput) *string { return o.ChecksumSHA256 },
|
||||||
|
setPart: func(p *types.CompletedPart, v *string) { p.ChecksumSHA256 = v },
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every part but the last has to reach the 5MB multipart minimum.
|
||||||
|
parts := [][]byte{bytes.Repeat([]byte("a"), 5*1024*1024), []byte("tail")}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(string(tc.algorithm), func(t *testing.T) {
|
||||||
|
key := "multipart-" + string(tc.algorithm)
|
||||||
|
create, err := client.CreateMultipartUpload(context.Background(), &s3.CreateMultipartUploadInput{
|
||||||
|
Bucket: aws.String(bucket),
|
||||||
|
Key: aws.String(key),
|
||||||
|
ChecksumAlgorithm: tc.algorithm,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, tc.algorithm, create.ChecksumAlgorithm)
|
||||||
|
require.Equal(t, tc.expectedType, create.ChecksumType)
|
||||||
|
|
||||||
|
var completed []types.CompletedPart
|
||||||
|
for i, data := range parts {
|
||||||
|
part, err := client.UploadPart(context.Background(), &s3.UploadPartInput{
|
||||||
|
Bucket: aws.String(bucket),
|
||||||
|
Key: aws.String(key),
|
||||||
|
UploadId: create.UploadId,
|
||||||
|
PartNumber: aws.Int32(int32(i + 1)),
|
||||||
|
Body: bytes.NewReader(data),
|
||||||
|
ChecksumAlgorithm: tc.algorithm,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotEmpty(t, aws.ToString(tc.partSum(part)))
|
||||||
|
|
||||||
|
entry := types.CompletedPart{ETag: part.ETag, PartNumber: aws.Int32(int32(i + 1))}
|
||||||
|
tc.setPart(&entry, tc.partSum(part))
|
||||||
|
completed = append(completed, entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
done, err := client.CompleteMultipartUpload(context.Background(), &s3.CompleteMultipartUploadInput{
|
||||||
|
Bucket: aws.String(bucket),
|
||||||
|
Key: aws.String(key),
|
||||||
|
UploadId: create.UploadId,
|
||||||
|
MultipartUpload: &types.CompletedMultipartUpload{Parts: completed},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotEmpty(t, aws.ToString(tc.completeSum(done)))
|
||||||
|
require.Equal(t, tc.expectedType, done.ChecksumType)
|
||||||
|
if tc.expectedType == types.ChecksumTypeComposite {
|
||||||
|
require.True(t, strings.HasSuffix(aws.ToString(tc.completeSum(done)), fmt.Sprintf("-%d", len(parts))))
|
||||||
|
}
|
||||||
|
|
||||||
|
head, err := client.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||||
|
Bucket: aws.String(bucket),
|
||||||
|
Key: aws.String(key),
|
||||||
|
ChecksumMode: types.ChecksumModeEnabled,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, aws.ToString(tc.completeSum(done)), aws.ToString(tc.headSum(head)))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -43,6 +43,10 @@ const (
|
|||||||
type InitiateMultipartUploadResult struct {
|
type InitiateMultipartUploadResult struct {
|
||||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"`
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"`
|
||||||
s3.CreateMultipartUploadOutput
|
s3.CreateMultipartUploadOutput
|
||||||
|
|
||||||
|
// Checksum fields — returned as HTTP response headers, not in the XML body
|
||||||
|
ChecksumAlgorithm string `xml:"-"`
|
||||||
|
ChecksumType string `xml:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// getRequestScheme determines the URL scheme (http or https) from the request
|
// getRequestScheme determines the URL scheme (http or https) from the request
|
||||||
@@ -148,6 +152,8 @@ func (s3a *S3ApiServer) createMultipartUpload(r *http.Request, input *s3.CreateM
|
|||||||
Key: objectKey(input.Key),
|
Key: objectKey(input.Key),
|
||||||
UploadId: aws.String(uploadIdString),
|
UploadId: aws.String(uploadIdString),
|
||||||
},
|
},
|
||||||
|
ChecksumAlgorithm: checksumAlgorithmNameFromHeaderName(checksumHeaderName),
|
||||||
|
ChecksumType: checksumType,
|
||||||
}
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -160,10 +166,8 @@ type CompleteMultipartUploadResult struct {
|
|||||||
Key *string `xml:"Key,omitempty"`
|
Key *string `xml:"Key,omitempty"`
|
||||||
ETag *string `xml:"ETag,omitempty"`
|
ETag *string `xml:"ETag,omitempty"`
|
||||||
|
|
||||||
// Checksum fields — returned as HTTP response headers, not in the XML body
|
ChecksumResult
|
||||||
ChecksumHeaderName string `xml:"-"`
|
ChecksumType string `xml:"ChecksumType,omitempty"`
|
||||||
ChecksumValue string `xml:"-"`
|
|
||||||
ChecksumType string `xml:"-"`
|
|
||||||
|
|
||||||
// VersionId is NOT included in XML body - it should only be in x-amz-version-id HTTP header
|
// VersionId is NOT included in XML body - it should only be in x-amz-version-id HTTP header
|
||||||
|
|
||||||
@@ -253,6 +257,8 @@ func completeMultipartResult(r *http.Request, input *s3.CompleteMultipartUploadI
|
|||||||
result.VersionId = aws.String(versionId)
|
result.VersionId = aws.String(versionId)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
result.SetChecksum(string(entry.Extended[s3_constants.ExtChecksumAlgorithm]), string(entry.Extended[s3_constants.ExtChecksumValue]))
|
||||||
|
result.ChecksumType = string(entry.Extended[s3_constants.ExtChecksumType])
|
||||||
}
|
}
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
@@ -763,15 +769,14 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
|||||||
// For versioned buckets, all content is stored in .versions directory
|
// For versioned buckets, all content is stored in .versions directory
|
||||||
// The latest version information is tracked in the .versions directory metadata
|
// The latest version information is tracked in the .versions directory metadata
|
||||||
output = &CompleteMultipartUploadResult{
|
output = &CompleteMultipartUploadResult{
|
||||||
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
||||||
Bucket: input.Bucket,
|
Bucket: input.Bucket,
|
||||||
ETag: aws.String(etagQuote),
|
ETag: aws.String(etagQuote),
|
||||||
Key: objectKey(input.Key),
|
Key: objectKey(input.Key),
|
||||||
VersionId: aws.String(versionId),
|
VersionId: aws.String(versionId),
|
||||||
ChecksumHeaderName: completionState.checksumHeaderName,
|
ChecksumType: completionState.checksumType,
|
||||||
ChecksumValue: completionState.checksumValue,
|
|
||||||
ChecksumType: completionState.checksumType,
|
|
||||||
}
|
}
|
||||||
|
output.SetChecksum(completionState.checksumHeaderName, completionState.checksumValue)
|
||||||
return s3err.ErrNone
|
return s3err.ErrNone
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -840,15 +845,14 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
|||||||
|
|
||||||
// Note: Suspended versioning should NOT return VersionId field according to AWS S3 spec
|
// Note: Suspended versioning should NOT return VersionId field according to AWS S3 spec
|
||||||
output = &CompleteMultipartUploadResult{
|
output = &CompleteMultipartUploadResult{
|
||||||
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
||||||
Bucket: input.Bucket,
|
Bucket: input.Bucket,
|
||||||
ETag: aws.String(etagQuote),
|
ETag: aws.String(etagQuote),
|
||||||
Key: objectKey(input.Key),
|
Key: objectKey(input.Key),
|
||||||
ChecksumHeaderName: completionState.checksumHeaderName,
|
ChecksumType: completionState.checksumType,
|
||||||
ChecksumValue: completionState.checksumValue,
|
|
||||||
ChecksumType: completionState.checksumType,
|
|
||||||
// VersionId field intentionally omitted for suspended versioning
|
// VersionId field intentionally omitted for suspended versioning
|
||||||
}
|
}
|
||||||
|
output.SetChecksum(completionState.checksumHeaderName, completionState.checksumValue)
|
||||||
return s3err.ErrNone
|
return s3err.ErrNone
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -911,14 +915,13 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
|||||||
|
|
||||||
// For non-versioned buckets, return response without VersionId
|
// For non-versioned buckets, return response without VersionId
|
||||||
output = &CompleteMultipartUploadResult{
|
output = &CompleteMultipartUploadResult{
|
||||||
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
||||||
Bucket: input.Bucket,
|
Bucket: input.Bucket,
|
||||||
ETag: aws.String(etagQuote),
|
ETag: aws.String(etagQuote),
|
||||||
Key: objectKey(input.Key),
|
Key: objectKey(input.Key),
|
||||||
ChecksumHeaderName: completionState.checksumHeaderName,
|
ChecksumType: completionState.checksumType,
|
||||||
ChecksumValue: completionState.checksumValue,
|
|
||||||
ChecksumType: completionState.checksumType,
|
|
||||||
}
|
}
|
||||||
|
output.SetChecksum(completionState.checksumHeaderName, completionState.checksumValue)
|
||||||
return s3err.ErrNone
|
return s3err.ErrNone
|
||||||
}
|
}
|
||||||
var finalizeCode s3err.ErrorCode
|
var finalizeCode s3err.ErrorCode
|
||||||
|
|||||||
@@ -4,11 +4,17 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"hash/crc32"
|
"hash/crc32"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
|
"github.com/aws/aws-sdk-go/service/s3"
|
||||||
"github.com/minio/crc64nvme"
|
"github.com/minio/crc64nvme"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||||
)
|
)
|
||||||
|
|
||||||
func makeCRC64NVMEPartEntry(data []byte) *filer_pb.Entry {
|
func makeCRC64NVMEPartEntry(data []byte) *filer_pb.Entry {
|
||||||
@@ -144,3 +150,57 @@ func TestResolveMultipartChecksumType(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CompleteMultipartUpload returns the object checksum in the XML body, where the
|
||||||
|
// AWS SDKs read it from — not as an HTTP response header.
|
||||||
|
func TestCompleteMultipartUploadResultChecksumXML(t *testing.T) {
|
||||||
|
result := &CompleteMultipartUploadResult{
|
||||||
|
ETag: aws.String("\"etag-1\""),
|
||||||
|
ChecksumType: s3_constants.ChecksumTypeComposite,
|
||||||
|
}
|
||||||
|
result.SetChecksum(s3_constants.AmzChecksumCRC32C, "fx4FQw==-1")
|
||||||
|
|
||||||
|
encoded := string(s3err.EncodeXMLResponse(result))
|
||||||
|
for _, want := range []string{
|
||||||
|
"<ChecksumCRC32C>fx4FQw==-1</ChecksumCRC32C>",
|
||||||
|
"<ChecksumType>COMPOSITE</ChecksumType>",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(encoded, want) {
|
||||||
|
t.Fatalf("response %q does not contain %q", encoded, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(encoded, "<ChecksumCRC32>") {
|
||||||
|
t.Fatalf("response %q carries an unrequested algorithm", encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A retried CompleteMultipartUpload rebuilds the response from the committed
|
||||||
|
// entry, so it must repeat the checksum the first attempt returned.
|
||||||
|
func TestCompleteMultipartResultChecksumFromEntry(t *testing.T) {
|
||||||
|
input := &s3.CompleteMultipartUploadInput{Bucket: aws.String("bucket"), Key: aws.String("key")}
|
||||||
|
entry := &filer_pb.Entry{Extended: map[string][]byte{
|
||||||
|
s3_constants.ExtChecksumAlgorithm: []byte(s3_constants.AmzChecksumCRC32C),
|
||||||
|
s3_constants.ExtChecksumValue: []byte("fx4FQw==-1"),
|
||||||
|
s3_constants.ExtChecksumType: []byte(s3_constants.ChecksumTypeComposite),
|
||||||
|
}}
|
||||||
|
|
||||||
|
result := completeMultipartResult(httptest.NewRequest(http.MethodPost, "/bucket/key", nil), input, "\"etag-1\"", entry)
|
||||||
|
if result.ChecksumCRC32C != "fx4FQw==-1" {
|
||||||
|
t.Fatalf("ChecksumCRC32C = %q, want %q", result.ChecksumCRC32C, "fx4FQw==-1")
|
||||||
|
}
|
||||||
|
if result.ChecksumType != s3_constants.ChecksumTypeComposite {
|
||||||
|
t.Fatalf("ChecksumType = %q, want %q", result.ChecksumType, s3_constants.ChecksumTypeComposite)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CreateMultipartUpload echoes the algorithm and type it recorded, so a client
|
||||||
|
// can tell which checksum the upload will be completed with.
|
||||||
|
func TestCreateMultipartUploadResultChecksumHeaders(t *testing.T) {
|
||||||
|
result := &InitiateMultipartUploadResult{
|
||||||
|
ChecksumAlgorithm: "CRC32C",
|
||||||
|
ChecksumType: s3_constants.ChecksumTypeComposite,
|
||||||
|
}
|
||||||
|
if encoded := string(s3err.EncodeXMLResponse(result)); strings.Contains(encoded, "Checksum") {
|
||||||
|
t.Fatalf("response %q carries checksum members in the XML body", encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -787,13 +787,9 @@ func pathToBucketObjectAndVersion(rawPath, decodedPath string) (bucket, object,
|
|||||||
}
|
}
|
||||||
|
|
||||||
type CopyPartResult struct {
|
type CopyPartResult struct {
|
||||||
LastModified time.Time `xml:"LastModified"`
|
LastModified time.Time `xml:"LastModified"`
|
||||||
ETag string `xml:"ETag"`
|
ETag string `xml:"ETag"`
|
||||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
ChecksumResult
|
||||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
|
||||||
ChecksumCRC64NVME string `xml:"ChecksumCRC64NVME,omitempty"`
|
|
||||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
|
||||||
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildCopyPartResult(etag string, lastModified time.Time, metadata SSEResponseMetadata) CopyPartResult {
|
func buildCopyPartResult(etag string, lastModified time.Time, metadata SSEResponseMetadata) CopyPartResult {
|
||||||
@@ -801,18 +797,7 @@ func buildCopyPartResult(etag string, lastModified time.Time, metadata SSERespon
|
|||||||
ETag: etag,
|
ETag: etag,
|
||||||
LastModified: lastModified,
|
LastModified: lastModified,
|
||||||
}
|
}
|
||||||
switch metadata.ChecksumHeaderName {
|
result.SetChecksum(metadata.ChecksumHeaderName, metadata.ChecksumValue)
|
||||||
case s3_constants.AmzChecksumCRC32:
|
|
||||||
result.ChecksumCRC32 = metadata.ChecksumValue
|
|
||||||
case s3_constants.AmzChecksumCRC32C:
|
|
||||||
result.ChecksumCRC32C = metadata.ChecksumValue
|
|
||||||
case s3_constants.AmzChecksumCRC64NVME:
|
|
||||||
result.ChecksumCRC64NVME = metadata.ChecksumValue
|
|
||||||
case s3_constants.AmzChecksumSHA1:
|
|
||||||
result.ChecksumSHA1 = metadata.ChecksumValue
|
|
||||||
case s3_constants.AmzChecksumSHA256:
|
|
||||||
result.ChecksumSHA256 = metadata.ChecksumValue
|
|
||||||
}
|
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ func TestBuildCopyPartResult(t *testing.T) {
|
|||||||
header: s3_constants.AmzChecksumCRC32,
|
header: s3_constants.AmzChecksumCRC32,
|
||||||
element: "<ChecksumCRC32>value</ChecksumCRC32>",
|
element: "<ChecksumCRC32>value</ChecksumCRC32>",
|
||||||
expected: CopyPartResult{
|
expected: CopyPartResult{
|
||||||
ETag: "etag", LastModified: modified, ChecksumCRC32: "value",
|
ETag: "etag", LastModified: modified, ChecksumResult: ChecksumResult{ChecksumCRC32: "value"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -83,7 +83,7 @@ func TestBuildCopyPartResult(t *testing.T) {
|
|||||||
header: s3_constants.AmzChecksumCRC32C,
|
header: s3_constants.AmzChecksumCRC32C,
|
||||||
element: "<ChecksumCRC32C>value</ChecksumCRC32C>",
|
element: "<ChecksumCRC32C>value</ChecksumCRC32C>",
|
||||||
expected: CopyPartResult{
|
expected: CopyPartResult{
|
||||||
ETag: "etag", LastModified: modified, ChecksumCRC32C: "value",
|
ETag: "etag", LastModified: modified, ChecksumResult: ChecksumResult{ChecksumCRC32C: "value"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -91,7 +91,7 @@ func TestBuildCopyPartResult(t *testing.T) {
|
|||||||
header: s3_constants.AmzChecksumCRC64NVME,
|
header: s3_constants.AmzChecksumCRC64NVME,
|
||||||
element: "<ChecksumCRC64NVME>value</ChecksumCRC64NVME>",
|
element: "<ChecksumCRC64NVME>value</ChecksumCRC64NVME>",
|
||||||
expected: CopyPartResult{
|
expected: CopyPartResult{
|
||||||
ETag: "etag", LastModified: modified, ChecksumCRC64NVME: "value",
|
ETag: "etag", LastModified: modified, ChecksumResult: ChecksumResult{ChecksumCRC64NVME: "value"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -99,7 +99,7 @@ func TestBuildCopyPartResult(t *testing.T) {
|
|||||||
header: s3_constants.AmzChecksumSHA1,
|
header: s3_constants.AmzChecksumSHA1,
|
||||||
element: "<ChecksumSHA1>value</ChecksumSHA1>",
|
element: "<ChecksumSHA1>value</ChecksumSHA1>",
|
||||||
expected: CopyPartResult{
|
expected: CopyPartResult{
|
||||||
ETag: "etag", LastModified: modified, ChecksumSHA1: "value",
|
ETag: "etag", LastModified: modified, ChecksumResult: ChecksumResult{ChecksumSHA1: "value"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -107,7 +107,7 @@ func TestBuildCopyPartResult(t *testing.T) {
|
|||||||
header: s3_constants.AmzChecksumSHA256,
|
header: s3_constants.AmzChecksumSHA256,
|
||||||
element: "<ChecksumSHA256>value</ChecksumSHA256>",
|
element: "<ChecksumSHA256>value</ChecksumSHA256>",
|
||||||
expected: CopyPartResult{
|
expected: CopyPartResult{
|
||||||
ETag: "etag", LastModified: modified, ChecksumSHA256: "value",
|
ETag: "etag", LastModified: modified, ChecksumResult: ChecksumResult{ChecksumSHA256: "value"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -120,6 +120,13 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if response.ChecksumAlgorithm != "" {
|
||||||
|
w.Header().Set(s3_constants.AmzChecksumAlgorithm, response.ChecksumAlgorithm)
|
||||||
|
if response.ChecksumType != "" {
|
||||||
|
w.Header().Set(s3_constants.AmzChecksumType, response.ChecksumType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
writeSuccessResponseXML(w, r, response)
|
writeSuccessResponseXML(w, r, response)
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -180,14 +187,6 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
|
|||||||
w.Header().Set("x-amz-version-id", *response.VersionId)
|
w.Header().Set("x-amz-version-id", *response.VersionId)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set checksum header if present
|
|
||||||
if response.ChecksumHeaderName != "" && response.ChecksumValue != "" {
|
|
||||||
w.Header().Set(response.ChecksumHeaderName, response.ChecksumValue)
|
|
||||||
if response.ChecksumType != "" {
|
|
||||||
w.Header().Set(s3_constants.AmzChecksumType, response.ChecksumType)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stats_collect.RecordBucketActiveTime(bucket)
|
stats_collect.RecordBucketActiveTime(bucket)
|
||||||
stats_collect.S3UploadedObjectsCounter.WithLabelValues(bucket).Inc()
|
stats_collect.S3UploadedObjectsCounter.WithLabelValues(bucket).Inc()
|
||||||
|
|
||||||
|
|||||||
@@ -1067,6 +1067,31 @@ var checksumHeaders = []struct {
|
|||||||
{s3_constants.AmzChecksumSHA256, ChecksumAlgorithmSHA256, s3_constants.AmzChecksumSHA256},
|
{s3_constants.AmzChecksumSHA256, ChecksumAlgorithmSHA256, s3_constants.AmzChecksumSHA256},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ChecksumResult carries the flexible-checksum members S3 returns inside an XML
|
||||||
|
// response body, keyed by the canonical x-amz-checksum-* header name.
|
||||||
|
type ChecksumResult struct {
|
||||||
|
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||||
|
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||||
|
ChecksumCRC64NVME string `xml:"ChecksumCRC64NVME,omitempty"`
|
||||||
|
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||||
|
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *ChecksumResult) SetChecksum(headerName, value string) {
|
||||||
|
switch headerName {
|
||||||
|
case s3_constants.AmzChecksumCRC32:
|
||||||
|
c.ChecksumCRC32 = value
|
||||||
|
case s3_constants.AmzChecksumCRC32C:
|
||||||
|
c.ChecksumCRC32C = value
|
||||||
|
case s3_constants.AmzChecksumCRC64NVME:
|
||||||
|
c.ChecksumCRC64NVME = value
|
||||||
|
case s3_constants.AmzChecksumSHA1:
|
||||||
|
c.ChecksumSHA1 = value
|
||||||
|
case s3_constants.AmzChecksumSHA256:
|
||||||
|
c.ChecksumSHA256 = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// lookupHeaderOrQuery returns the value of an x-amz-* parameter, checking the
|
// lookupHeaderOrQuery returns the value of an x-amz-* parameter, checking the
|
||||||
// request headers first and falling back to the pre-parsed query values. AWS
|
// request headers first and falling back to the pre-parsed query values. AWS
|
||||||
// SDK presigners hoist headers such as x-amz-sdk-checksum-algorithm into the
|
// SDK presigners hoist headers such as x-amz-sdk-checksum-algorithm into the
|
||||||
|
|||||||
Reference in New Issue
Block a user