From 3810fb112341cfac9ff982bb9b0f9b759646036f Mon Sep 17 00:00:00 2001 From: Mohd Quamar Tyagi <104281681+Tyagiquamar@users.noreply.github.com> Date: Sat, 24 Oct 2026 06:51:26 +0530 Subject: [PATCH] s3: url-encode ListMultipartUploads results for encoding-type=url (#11998) * s3: url-encode ListMultipartUploads results for encoding-type=url The response echoed EncodingType=url but returned raw upload keys, and XML 1.0 cannot carry control characters, so the encoder replaced them with U+FFFD. The listed key then matched no upload: AbortMultipartUpload with it answered 404 NoSuchUpload, leaving such uploads uncleanable. Encode the key-named elements AWS encodes for this API (Key, Prefix, Delimiter, KeyMarker, NextKeyMarker) the way ListObjects encodes its keys when encoding-type=url is requested. Upload ids stay opaque, and responses without encoding-type=url are unchanged. Fixes #11991 * s3: echo EncodingType only when encoding-type=url is honored A request with an unrecognized encoding-type echoed the value back in while the keys stayed raw, matching neither AWS (which omits the element unless url encoding applies) nor the ListObjects handlers, which gate the echo on the parsed url flag. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: chrislusf Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/s3api/filer_multipart.go | 28 ++++++- .../filer_multipart_list_encoding_test.go | 76 +++++++++++++++++++ weed/s3api/s3api_object_handlers_multipart.go | 2 - 3 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 weed/s3api/filer_multipart_list_encoding_test.go diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 6f22429fb6..9d9f1f8cc8 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -1400,7 +1400,6 @@ func (p *multipartUploadPage) result() *ListMultipartUploadsResult { output := &ListMultipartUploadsResult{ Bucket: input.Bucket, Delimiter: input.Delimiter, - EncodingType: input.EncodingType, KeyMarker: input.KeyMarker, MaxUploads: aws.Int64(int64(p.limit)), Prefix: input.Prefix, @@ -1425,9 +1424,36 @@ func (p *multipartUploadPage) result() *ListMultipartUploadsResult { output.IsTruncated = aws.Bool(true) output.NextKeyMarker, output.NextUploadIdMarker = last.Key, last.UploadId } + if aws.StringValue(input.EncodingType) == s3.EncodingTypeUrl { + // AWS url-encodes these key-named elements when encoding-type=url is + // requested; without it XML 1.0 cannot carry the keys at all and the + // encoder replaces their characters with U+FFFD. + output.EncodingType = input.EncodingType + output.Delimiter = encodeMultipartUploadURLField(output.Delimiter) + output.Prefix = encodeMultipartUploadURLField(output.Prefix) + output.KeyMarker = encodeMultipartUploadURLField(output.KeyMarker) + output.NextKeyMarker = encodeMultipartUploadURLField(output.NextKeyMarker) + for _, upload := range output.Upload { + upload.Key = encodeMultipartUploadURLField(upload.Key) + } + } return output } +// encodeMultipartUploadURLField url-encodes key text the way ListObjects does: +// every "/"-separated segment is query-escaped and the separators are kept, so a +// client recovers the exact key with url.QueryUnescape. +func encodeMultipartUploadURLField(value *string) *string { + if value == nil { + return nil + } + segments := strings.Split(*value, "/") + for i, segment := range segments { + segments[i] = url.QueryEscape(segment) + } + return aws.String(strings.Join(segments, "/")) +} + type ListPartsResult struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"` diff --git a/weed/s3api/filer_multipart_list_encoding_test.go b/weed/s3api/filer_multipart_list_encoding_test.go new file mode 100644 index 0000000000..2449c10585 --- /dev/null +++ b/weed/s3api/filer_multipart_list_encoding_test.go @@ -0,0 +1,76 @@ +package s3api + +import ( + "encoding/xml" + "net/url" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// With encoding-type=url the upload keys have to be url-encoded, as ListObjects +// does; otherwise XML 1.0 cannot carry a key with a control character, the +// encoder turns it into U+FFFD, and no client can recover the key to abort the +// upload. +func TestSelectMultipartUploadsEncodingTypeURL(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "m\x01p b+.bin", 10), + uploadEntry("2", "plain/key.txt", 20), + } + input := listUploadsInput("m\x01", "m\x01", "", 1000) + input.EncodingType = aws.String(s3.EncodingTypeUrl) + out := selectMultipartUploads(input, entries) + + require.Len(t, out.Upload, 1) + assert.Equal(t, "m%01p+b%2B.bin", *out.Upload[0].Key) + decoded, err := url.QueryUnescape(*out.Upload[0].Key) + require.NoError(t, err) + assert.Equal(t, "m\x01p b+.bin", decoded) + assert.Equal(t, "m%01", *out.Prefix) + assert.Equal(t, "m%01", *out.KeyMarker) + + body, err := xml.Marshal(out) + require.NoError(t, err) + assert.NotContains(t, string(body), "\uFFFD") + assert.Contains(t, string(body), "m%01p+b%2B.bin") + assert.Contains(t, string(body), "url") +} + +func TestSelectMultipartUploadsEncodingTypeURLNextKeyMarker(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "a\x1fb", 10), + uploadEntry("2", "c", 20), + } + input := listUploadsInput("", "", "", 1) + input.EncodingType = aws.String(s3.EncodingTypeUrl) + out := selectMultipartUploads(input, entries) + + require.True(t, *out.IsTruncated) + assert.Equal(t, "a%1Fb", *out.NextKeyMarker) + // The upload id is opaque, not a key name. + assert.Equal(t, "1", *out.NextUploadIdMarker) + + body, err := xml.Marshal(out) + require.NoError(t, err) + assert.NotContains(t, string(body), "\uFFFD") +} + +// Without encoding-type=url the response is unchanged, and the keys keep their +// raw form for clients that do not ask for encoding. +func TestSelectMultipartUploadsKeepsKeysRawWithoutEncodingType(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "m\x01p b+.bin", 10), + uploadEntry("2", "plain/key.txt", 20), + } + out := selectMultipartUploads(listUploadsInput("", "", "", 1000), entries) + + var got []string + for _, upload := range out.Upload { + got = append(got, *upload.Key) + } + assert.Equal(t, []string{"m\x01p b+.bin", "plain/key.txt"}, got) +} diff --git a/weed/s3api/s3api_object_handlers_multipart.go b/weed/s3api/s3api_object_handlers_multipart.go index 9a98da6627..945d7862d6 100644 --- a/weed/s3api/s3api_object_handlers_multipart.go +++ b/weed/s3api/s3api_object_handlers_multipart.go @@ -290,8 +290,6 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht return } - // TODO handle encodingType - writeSuccessResponseXML(w, r, response) }