From 1fd5008d63081eb295fe4b6496edc96cf0bc266a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 18 Nov 2026 15:55:41 -0800 Subject: [PATCH] s3api: listing argument validation and encoding-type=url for versions, prefix and delimiter (#12753) * s3api: reject an empty ListObjectsV2 continuation-token AWS answers continuation-token= with 400 InvalidArgument ("The continuation token provided is incorrect"); SeaweedFS listed from the start of the bucket. No token this server issues is empty, and raw-key and base64: tokens keep resuming as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3api: reject an encoding-type other than url on listings AWS answers any encoding-type other than url with 400 InvalidArgument ("Invalid Encoding Method specified in Request") on ListObjects, ListObjectsV2, ListObjectVersions, ListMultipartUploads and ListParts; SeaweedFS ignored the value and listed unencoded keys. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3api: apply encoding-type=url to ListObjectVersions The handler accepted encoding-type=url but returned raw keys. It now URL-encodes Key in Version and DeleteMarker entries, CommonPrefixes, Prefix, Delimiter, KeyMarker and NextKeyMarker with the urlPathEscape form ListObjects uses, and echoes EncodingType. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3api: URL-encode Prefix and Delimiter in ListObjects responses Under encoding-type=url, ListObjects and ListObjectsV2 encoded keys, CommonPrefixes, markers and StartAfter but echoed Prefix and Delimiter raw, so a client decoding them once corrupted a prefix holding '%' or '+'. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK --------- Co-authored-by: Claude Opus 5.5 --- weed/s3api/s3api_object_handlers_list.go | 31 ++++- .../s3api_object_handlers_list_args_test.go | 111 ++++++++++++++++++ weed/s3api/s3api_object_handlers_multipart.go | 6 + weed/s3api/s3api_object_versioning.go | 29 +++++ weed/s3api/s3err/s3api_errors.go | 6 + 5 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 weed/s3api/s3api_object_handlers_list_args_test.go diff --git a/weed/s3api/s3api_object_handlers_list.go b/weed/s3api/s3api_object_handlers_list.go index 0c10c1795d..e90228f52c 100644 --- a/weed/s3api/s3api_object_handlers_list.go +++ b/weed/s3api/s3api_object_handlers_list.go @@ -69,6 +69,10 @@ type listBucketResultV1 struct { } func toListBucketResultV1(in ListBucketResult) listBucketResultV1 { + if in.EncodingType == s3.EncodingTypeUrl { + in.Prefix = urlPathEscape(in.Prefix) + in.Delimiter = urlPathEscape(in.Delimiter) + } return listBucketResultV1{ Metadata: in.Metadata, Name: in.Name, @@ -155,6 +159,8 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ if encodingTypeUrl { responseV2.EncodingType = s3.EncodingTypeUrl responseV2.StartAfter = urlPathEscape(startAfter) + responseV2.Prefix = urlPathEscape(response.Prefix) + responseV2.Delimiter = urlPathEscape(response.Delimiter) } glog.V(3).Infof("ListObjectsV2Handler response: %+v", responseV2) @@ -1609,9 +1615,16 @@ func (s3a *S3ApiServer) dirHoldsOnlyHiddenEntriesInner(ctx context.Context, clie func getListObjectsV2Args(values url.Values) (prefix, startAfter, delimiter string, token OptionalString, encodingTypeUrl bool, fetchOwner bool, maxkeys uint16, allowUnordered bool, errCode s3err.ErrorCode) { prefix = values.Get("prefix") token = OptionalString{set: values.Has("continuation-token"), string: values.Get("continuation-token")} + // Every NextContinuationToken this server writes is non-empty. + if token.set && token.string == "" { + errCode = s3err.ErrInvalidContinuationToken + return + } startAfter = values.Get("start-after") delimiter = values.Get("delimiter") - encodingTypeUrl = values.Get("encoding-type") == s3.EncodingTypeUrl + if encodingTypeUrl, errCode = parseEncodingType(values); errCode != s3err.ErrNone { + return + } maxkeys, errCode = parseMaxKeys(values.Get("max-keys")) if errCode != s3err.ErrNone { return @@ -1626,7 +1639,9 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, prefix = values.Get("prefix") marker = values.Get("marker") delimiter = values.Get("delimiter") - encodingTypeUrl = values.Get("encoding-type") == "url" + if encodingTypeUrl, errCode = parseEncodingType(values); errCode != s3err.ErrNone { + return + } maxkeys, errCode = parseMaxKeys(values.Get("max-keys")) if errCode != s3err.ErrNone { return @@ -1636,6 +1651,18 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, return } +// parseEncodingType reports whether encoding-type=url was requested. AWS +// accepts no other encoding and answers anything else with InvalidArgument. +func parseEncodingType(values url.Values) (bool, s3err.ErrorCode) { + switch values.Get("encoding-type") { + case "": + return false, s3err.ErrNone + case s3.EncodingTypeUrl: + return true, s3err.ErrNone + } + return false, s3err.ErrInvalidEncodingMethod +} + // parseMaxKeys parses the max-keys query parameter. Like AWS S3, any integer in // [0, 2147483647] is accepted and the page size is capped at maxObjectListSizeLimit; // an empty value means the default limit, anything else is ErrInvalidMaxKeys. diff --git a/weed/s3api/s3api_object_handlers_list_args_test.go b/weed/s3api/s3api_object_handlers_list_args_test.go new file mode 100644 index 0000000000..f648a1fa71 --- /dev/null +++ b/weed/s3api/s3api_object_handlers_list_args_test.go @@ -0,0 +1,111 @@ +package s3api + +import ( + "encoding/xml" + "net/url" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetListObjectsV2ArgsContinuationToken(t *testing.T) { + cases := []struct { + query string + want string + wantErr s3err.ErrorCode + }{ + {query: "continuation-token=", wantErr: s3err.ErrInvalidContinuationToken}, + {query: "continuation-token=photos%2F2006%2FJanuary%2Fsample.jpg", want: "photos/2006/January/sample.jpg"}, + {query: "continuation-token=" + encodeListContinuationToken("ctrl/\x01key"), want: "ctrl/\x01key"}, + {query: "continuation-token=base64%3Aplain", want: "base64:plain"}, + {query: "start-after=a"}, + } + for _, tc := range cases { + values, err := url.ParseQuery(tc.query) + require.NoError(t, err) + _, _, _, token, _, _, _, _, errCode := getListObjectsV2Args(values) + assert.Equal(t, tc.wantErr, errCode, tc.query) + if tc.wantErr == s3err.ErrNone { + assert.Equal(t, tc.want, decodeListContinuationToken(token.string), tc.query) + } + } +} + +func TestListArgsEncodingType(t *testing.T) { + cases := []struct { + query string + wantURL bool + wantErr s3err.ErrorCode + }{ + {query: ""}, + {query: "encoding-type=url", wantURL: true}, + {query: "encoding-type=URL", wantErr: s3err.ErrInvalidEncodingMethod}, + {query: "encoding-type=bogus", wantErr: s3err.ErrInvalidEncodingMethod}, + } + for _, tc := range cases { + values, err := url.ParseQuery(tc.query) + require.NoError(t, err) + + _, _, _, _, urlV2, _, _, _, errV2 := getListObjectsV2Args(values) + assert.Equal(t, tc.wantErr, errV2, "v2 %q", tc.query) + assert.Equal(t, tc.wantURL, urlV2, "v2 %q", tc.query) + + _, _, _, urlV1, _, _, errV1 := getListObjectsV1Args(values) + assert.Equal(t, tc.wantErr, errV1, "v1 %q", tc.query) + assert.Equal(t, tc.wantURL, urlV1, "v1 %q", tc.query) + + _, _, _, _, _, _, errUploads := getBucketMultipartResources(values) + assert.Equal(t, tc.wantErr, errUploads, "uploads %q", tc.query) + + _, _, _, _, errParts := getObjectResources(values) + assert.Equal(t, tc.wantErr, errParts, "parts %q", tc.query) + } +} + +func TestListObjectVersionsEncodeURL(t *testing.T) { + result := &S3ListObjectVersionsResult{ + Name: "b", + Prefix: "dir a/", + Delimiter: "/", + KeyMarker: encodeListContinuationToken("dir a/café+1"), + NextKeyMarker: encodeListContinuationToken("dir a/100%"), + Entries: []VersionListEntry{ + {Version: &VersionEntry{Key: "dir a/café+1"}}, + {DeleteMarker: &DeleteMarkerEntry{Key: "dir a/100%"}}, + {Prefix: &PrefixEntry{Prefix: "dir a/sub dir/"}}, + }, + } + result.encodeURL() + + out, err := xml.Marshal(result) + require.NoError(t, err) + body := string(out) + for _, want := range []string{ + "dir%20a/", + "/", + "dir%20a/caf%C3%A9%2B1", + "dir%20a/100%25", + "dir%20a/caf%C3%A9%2B1", + "dir%20a/100%25", + "dir%20a/sub%20dir/", + "url", + } { + assert.Contains(t, body, want) + } + decoded, err := url.PathUnescape("dir%20a/caf%C3%A9%2B1") + require.NoError(t, err) + assert.Equal(t, "dir a/café+1", decoded) +} + +func TestListObjectsV1EncodesPrefixAndDelimiter(t *testing.T) { + response := ListBucketResult{Name: "b", Prefix: "dir a/café+", Delimiter: "/", MaxKeys: 1000, EncodingType: "url"} + body := string(s3err.EncodeXMLResponse(toListBucketResultV1(response))) + assert.Contains(t, body, "dir%20a/caf%C3%A9%2B") + assert.Contains(t, body, "/") + + response.EncodingType = "" + body = string(s3err.EncodeXMLResponse(toListBucketResultV1(response))) + assert.Contains(t, body, "dir a/café+") +} diff --git a/weed/s3api/s3api_object_handlers_multipart.go b/weed/s3api/s3api_object_handlers_multipart.go index 1aa504704e..6a9458a4f6 100644 --- a/weed/s3api/s3api_object_handlers_multipart.go +++ b/weed/s3api/s3api_object_handlers_multipart.go @@ -657,6 +657,9 @@ func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadID maxUploads = maxUploadsList } encodingType = values.Get("encoding-type") + if _, encodingErr := parseEncodingType(values); encodingErr != s3err.ErrNone && errCode == s3err.ErrNone { + errCode = encodingErr + } return } @@ -680,6 +683,9 @@ func getObjectResources(values url.Values) (uploadID string, partNumberMarker, m maxParts = maxPartsList } encodingType = values.Get("encoding-type") + if _, encodingErr := parseEncodingType(values); encodingErr != s3err.ErrNone && errCode == s3err.ErrNone { + errCode = encodingErr + } return } diff --git a/weed/s3api/s3api_object_versioning.go b/weed/s3api/s3api_object_versioning.go index d2bc38a515..ff1ea2d3fd 100644 --- a/weed/s3api/s3api_object_versioning.go +++ b/weed/s3api/s3api_object_versioning.go @@ -18,6 +18,7 @@ import ( "sync" "time" + "github.com/aws/aws-sdk-go/service/s3" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -494,6 +495,26 @@ func (s3a *S3ApiServer) truncateAndSetMarkers(combinedList []versionListItem, ma return combinedList, nextKeyMarker, nextVersionIdMarker, isTruncated } +// encodeURL applies encoding-type=url to every key-bearing field, the form +// ListObjects uses for its keys; clients decode them once. +func (result *S3ListObjectVersionsResult) encodeURL() { + result.EncodingType = s3.EncodingTypeUrl + result.Prefix = urlPathEscape(result.Prefix) + result.Delimiter = urlPathEscape(result.Delimiter) + result.KeyMarker = urlPathEscape(result.KeyMarker) + result.NextKeyMarker = urlPathEscape(result.NextKeyMarker) + for _, e := range result.Entries { + switch { + case e.Version != nil: + e.Version.Key = urlPathEscape(e.Version.Key) + case e.DeleteMarker != nil: + e.DeleteMarker.Key = urlPathEscape(e.DeleteMarker.Key) + case e.Prefix != nil: + e.Prefix.Prefix = urlPathEscape(e.Prefix.Prefix) + } + } +} + // splitIntoResult builds the final S3ListObjectVersionsResult from the combined list. // It populates a single Entries slice that preserves the interleaved sort order. func (s3a *S3ApiServer) splitIntoResult(combinedList []versionListItem, bucket, prefix, keyMarker, versionIdMarker, delimiter string, maxKeys int, isTruncated bool, nextKeyMarker, nextVersionIdMarker string) *S3ListObjectVersionsResult { @@ -1991,6 +2012,11 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http versionIdMarker := query.Get("version-id-marker") delimiter := query.Get("delimiter") + encodingTypeUrl, errCode := parseEncodingType(query) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } maxKeys, errCode := parseMaxKeys(query.Get("max-keys")) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) @@ -2007,6 +2033,9 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http // Set the original prefix in the response (not the normalized internal prefix) result.Prefix = originalPrefix + if encodingTypeUrl { + result.encodeURL() + } glog.V(3).Infof("ListObjectVersionsHandler response: %+v", result) writeSuccessResponseXML(w, r, result) diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go index 67358067d0..6b8c4b74ae 100644 --- a/weed/s3api/s3err/s3api_errors.go +++ b/weed/s3api/s3err/s3api_errors.go @@ -68,6 +68,7 @@ const ( ErrInvalidMaxKeys ErrInvalidMaxBuckets ErrInvalidContinuationToken + ErrInvalidEncodingMethod ErrInvalidMaxUploads ErrInvalidMaxParts ErrInvalidMaxDeleteObjects @@ -257,6 +258,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "The continuation token provided is incorrect", HTTPStatusCode: http.StatusBadRequest, }, + ErrInvalidEncodingMethod: { + Code: "InvalidArgument", + Description: "Invalid Encoding Method specified in Request", + HTTPStatusCode: http.StatusBadRequest, + }, ErrInvalidMaxParts: { Code: "InvalidArgument", Description: "Argument max-parts must be an integer between 0 and 2147483647",