From d2a0002869f1b16c675b1c4d88f70ff712b74f53 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 18 Nov 2026 19:34:57 -0800 Subject: [PATCH] s3api: fix listing regressions from argument validation (empty continuation-token, V1 Prefix encoding) (#12760) * s3api: accept an empty ListObjectsV2 continuation-token An empty continuation-token lists from the start and is echoed back empty, as S3 does; rejecting it broke s3-tests test_bucket_listv2_continuationtoken_empty. Co-Authored-By: Claude Opus 5.5 * s3api: leave the ListObjects V1 Prefix unencoded under encoding-type=url S3 encodes the V1 Delimiter, markers and keys but echoes Prefix as sent, and botocore, which always asks for url encoding, decodes the V1 Prefix only for V2. Encoding it broke s3-tests test_bucket_list_prefix_unreadable. Co-Authored-By: Claude Opus 5.5 * s3api: use a codespell-clean key in the versions encoding test Co-Authored-By: Claude Opus 5.5 * s3api: URL-encode a V1 Prefix that XML cannot carry Leaving the V1 Prefix raw under encoding-type=url turned a control character such as \x01 into U+FFFD in the response. Such a prefix is URL-encoded again; prefixes that are valid XML text, including '\n', stay as sent. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- weed/s3api/s3api_object_handlers_list.go | 10 +++---- .../s3api_object_handlers_list_args_test.go | 29 +++++++++++-------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/weed/s3api/s3api_object_handlers_list.go b/weed/s3api/s3api_object_handlers_list.go index e90228f52c..f0eb1ced07 100644 --- a/weed/s3api/s3api_object_handlers_list.go +++ b/weed/s3api/s3api_object_handlers_list.go @@ -70,7 +70,10 @@ type listBucketResultV1 struct { func toListBucketResultV1(in ListBucketResult) listBucketResultV1 { if in.EncodingType == s3.EncodingTypeUrl { - in.Prefix = urlPathEscape(in.Prefix) + // botocore does not decode the V1 Prefix, so it stays raw unless XML cannot carry it. + if !isXMLText(in.Prefix) { + in.Prefix = urlPathEscape(in.Prefix) + } in.Delimiter = urlPathEscape(in.Delimiter) } return listBucketResultV1{ @@ -1615,11 +1618,6 @@ func (s3a *S3ApiServer) dirHoldsOnlyHiddenEntriesInner(ctx context.Context, clie func getListObjectsV2Args(values url.Values) (prefix, startAfter, delimiter string, token OptionalString, encodingTypeUrl bool, fetchOwner bool, maxkeys uint16, allowUnordered bool, errCode s3err.ErrorCode) { prefix = values.Get("prefix") token = OptionalString{set: values.Has("continuation-token"), string: values.Get("continuation-token")} - // Every NextContinuationToken this server writes is non-empty. - if token.set && token.string == "" { - errCode = s3err.ErrInvalidContinuationToken - return - } startAfter = values.Get("start-after") delimiter = values.Get("delimiter") if encodingTypeUrl, errCode = parseEncodingType(values); errCode != s3err.ErrNone { diff --git a/weed/s3api/s3api_object_handlers_list_args_test.go b/weed/s3api/s3api_object_handlers_list_args_test.go index f648a1fa71..582230b263 100644 --- a/weed/s3api/s3api_object_handlers_list_args_test.go +++ b/weed/s3api/s3api_object_handlers_list_args_test.go @@ -16,7 +16,7 @@ func TestGetListObjectsV2ArgsContinuationToken(t *testing.T) { want string wantErr s3err.ErrorCode }{ - {query: "continuation-token=", wantErr: s3err.ErrInvalidContinuationToken}, + {query: "continuation-token=&start-after=a", want: ""}, {query: "continuation-token=photos%2F2006%2FJanuary%2Fsample.jpg", want: "photos/2006/January/sample.jpg"}, {query: "continuation-token=" + encodeListContinuationToken("ctrl/\x01key"), want: "ctrl/\x01key"}, {query: "continuation-token=base64%3Aplain", want: "base64:plain"}, @@ -28,6 +28,7 @@ func TestGetListObjectsV2ArgsContinuationToken(t *testing.T) { _, _, _, token, _, _, _, _, errCode := getListObjectsV2Args(values) assert.Equal(t, tc.wantErr, errCode, tc.query) if tc.wantErr == s3err.ErrNone { + assert.Equal(t, values.Has("continuation-token"), token.set, tc.query) assert.Equal(t, tc.want, decodeListContinuationToken(token.string), tc.query) } } @@ -69,10 +70,10 @@ func TestListObjectVersionsEncodeURL(t *testing.T) { Name: "b", Prefix: "dir a/", Delimiter: "/", - KeyMarker: encodeListContinuationToken("dir a/café+1"), + KeyMarker: encodeListContinuationToken("dir a/résumé+1"), NextKeyMarker: encodeListContinuationToken("dir a/100%"), Entries: []VersionListEntry{ - {Version: &VersionEntry{Key: "dir a/café+1"}}, + {Version: &VersionEntry{Key: "dir a/résumé+1"}}, {DeleteMarker: &DeleteMarkerEntry{Key: "dir a/100%"}}, {Prefix: &PrefixEntry{Prefix: "dir a/sub dir/"}}, }, @@ -85,27 +86,31 @@ func TestListObjectVersionsEncodeURL(t *testing.T) { for _, want := range []string{ "dir%20a/", "/", - "dir%20a/caf%C3%A9%2B1", + "dir%20a/r%C3%A9sum%C3%A9%2B1", "dir%20a/100%25", - "dir%20a/caf%C3%A9%2B1", + "dir%20a/r%C3%A9sum%C3%A9%2B1", "dir%20a/100%25", "dir%20a/sub%20dir/", "url", } { assert.Contains(t, body, want) } - decoded, err := url.PathUnescape("dir%20a/caf%C3%A9%2B1") + decoded, err := url.PathUnescape("dir%20a/r%C3%A9sum%C3%A9%2B1") require.NoError(t, err) - assert.Equal(t, "dir a/café+1", decoded) + assert.Equal(t, "dir a/résumé+1", decoded) } -func TestListObjectsV1EncodesPrefixAndDelimiter(t *testing.T) { - response := ListBucketResult{Name: "b", Prefix: "dir a/café+", Delimiter: "/", MaxKeys: 1000, EncodingType: "url"} +func TestListObjectsV1EncodesDelimiterNotPrefix(t *testing.T) { + response := ListBucketResult{Name: "b", Prefix: "dir a/\n+", Delimiter: " ", MaxKeys: 1000, EncodingType: "url"} body := string(s3err.EncodeXMLResponse(toListBucketResultV1(response))) - assert.Contains(t, body, "dir%20a/caf%C3%A9%2B") - assert.Contains(t, body, "/") + assert.Contains(t, body, "dir a/ +") + assert.Contains(t, body, "%20") + + response.Prefix = "dir a/\x01" + body = string(s3err.EncodeXMLResponse(toListBucketResultV1(response))) + assert.Contains(t, body, "dir%20a/%01") response.EncodingType = "" body = string(s3err.EncodeXMLResponse(toListBucketResultV1(response))) - assert.Contains(t, body, "dir a/café+") + assert.Contains(t, body, " ") }