diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 6f22429fb6..9d9f1f8cc8 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -1400,7 +1400,6 @@ func (p *multipartUploadPage) result() *ListMultipartUploadsResult { output := &ListMultipartUploadsResult{ Bucket: input.Bucket, Delimiter: input.Delimiter, - EncodingType: input.EncodingType, KeyMarker: input.KeyMarker, MaxUploads: aws.Int64(int64(p.limit)), Prefix: input.Prefix, @@ -1425,9 +1424,36 @@ func (p *multipartUploadPage) result() *ListMultipartUploadsResult { output.IsTruncated = aws.Bool(true) output.NextKeyMarker, output.NextUploadIdMarker = last.Key, last.UploadId } + if aws.StringValue(input.EncodingType) == s3.EncodingTypeUrl { + // AWS url-encodes these key-named elements when encoding-type=url is + // requested; without it XML 1.0 cannot carry the keys at all and the + // encoder replaces their characters with U+FFFD. + output.EncodingType = input.EncodingType + output.Delimiter = encodeMultipartUploadURLField(output.Delimiter) + output.Prefix = encodeMultipartUploadURLField(output.Prefix) + output.KeyMarker = encodeMultipartUploadURLField(output.KeyMarker) + output.NextKeyMarker = encodeMultipartUploadURLField(output.NextKeyMarker) + for _, upload := range output.Upload { + upload.Key = encodeMultipartUploadURLField(upload.Key) + } + } return output } +// encodeMultipartUploadURLField url-encodes key text the way ListObjects does: +// every "/"-separated segment is query-escaped and the separators are kept, so a +// client recovers the exact key with url.QueryUnescape. +func encodeMultipartUploadURLField(value *string) *string { + if value == nil { + return nil + } + segments := strings.Split(*value, "/") + for i, segment := range segments { + segments[i] = url.QueryEscape(segment) + } + return aws.String(strings.Join(segments, "/")) +} + type ListPartsResult struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"` diff --git a/weed/s3api/filer_multipart_list_encoding_test.go b/weed/s3api/filer_multipart_list_encoding_test.go new file mode 100644 index 0000000000..2449c10585 --- /dev/null +++ b/weed/s3api/filer_multipart_list_encoding_test.go @@ -0,0 +1,76 @@ +package s3api + +import ( + "encoding/xml" + "net/url" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// With encoding-type=url the upload keys have to be url-encoded, as ListObjects +// does; otherwise XML 1.0 cannot carry a key with a control character, the +// encoder turns it into U+FFFD, and no client can recover the key to abort the +// upload. +func TestSelectMultipartUploadsEncodingTypeURL(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "m\x01p b+.bin", 10), + uploadEntry("2", "plain/key.txt", 20), + } + input := listUploadsInput("m\x01", "m\x01", "", 1000) + input.EncodingType = aws.String(s3.EncodingTypeUrl) + out := selectMultipartUploads(input, entries) + + require.Len(t, out.Upload, 1) + assert.Equal(t, "m%01p+b%2B.bin", *out.Upload[0].Key) + decoded, err := url.QueryUnescape(*out.Upload[0].Key) + require.NoError(t, err) + assert.Equal(t, "m\x01p b+.bin", decoded) + assert.Equal(t, "m%01", *out.Prefix) + assert.Equal(t, "m%01", *out.KeyMarker) + + body, err := xml.Marshal(out) + require.NoError(t, err) + assert.NotContains(t, string(body), "\uFFFD") + assert.Contains(t, string(body), "m%01p+b%2B.bin") + assert.Contains(t, string(body), "url") +} + +func TestSelectMultipartUploadsEncodingTypeURLNextKeyMarker(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "a\x1fb", 10), + uploadEntry("2", "c", 20), + } + input := listUploadsInput("", "", "", 1) + input.EncodingType = aws.String(s3.EncodingTypeUrl) + out := selectMultipartUploads(input, entries) + + require.True(t, *out.IsTruncated) + assert.Equal(t, "a%1Fb", *out.NextKeyMarker) + // The upload id is opaque, not a key name. + assert.Equal(t, "1", *out.NextUploadIdMarker) + + body, err := xml.Marshal(out) + require.NoError(t, err) + assert.NotContains(t, string(body), "\uFFFD") +} + +// Without encoding-type=url the response is unchanged, and the keys keep their +// raw form for clients that do not ask for encoding. +func TestSelectMultipartUploadsKeepsKeysRawWithoutEncodingType(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "m\x01p b+.bin", 10), + uploadEntry("2", "plain/key.txt", 20), + } + out := selectMultipartUploads(listUploadsInput("", "", "", 1000), entries) + + var got []string + for _, upload := range out.Upload { + got = append(got, *upload.Key) + } + assert.Equal(t, []string{"m\x01p b+.bin", "plain/key.txt"}, got) +} diff --git a/weed/s3api/s3api_object_handlers_multipart.go b/weed/s3api/s3api_object_handlers_multipart.go index 9a98da6627..945d7862d6 100644 --- a/weed/s3api/s3api_object_handlers_multipart.go +++ b/weed/s3api/s3api_object_handlers_multipart.go @@ -290,8 +290,6 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht return } - // TODO handle encodingType - writeSuccessResponseXML(w, r, response) }