mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
s3api: fix listing regressions from argument validation (empty continuation-token, V1 Prefix encoding) (#12760)
* s3api: accept an empty ListObjectsV2 continuation-token An empty continuation-token lists from the start and is echoed back empty, as S3 does; rejecting it broke s3-tests test_bucket_listv2_continuationtoken_empty. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3api: leave the ListObjects V1 Prefix unencoded under encoding-type=url S3 encodes the V1 Delimiter, markers and keys but echoes Prefix as sent, and botocore, which always asks for url encoding, decodes the V1 Prefix only for V2. Encoding it broke s3-tests test_bucket_list_prefix_unreadable. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3api: use a codespell-clean key in the versions encoding test Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3api: URL-encode a V1 Prefix that XML cannot carry Leaving the V1 Prefix raw under encoding-type=url turned a control character such as \x01 into U+FFFD in the response. Such a prefix is URL-encoded again; prefixes that are valid XML text, including '\n', stay as sent. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c01e87689e
commit
d2a0002869
2 files changed
+21
-18
No files matched your search
@@ -70,7 +70,10 @@ type listBucketResultV1 struct {
|
||||
|
||||
func toListBucketResultV1(in ListBucketResult) listBucketResultV1 {
|
||||
if in.EncodingType == s3.EncodingTypeUrl {
|
||||
in.Prefix = urlPathEscape(in.Prefix)
|
||||
// botocore does not decode the V1 Prefix, so it stays raw unless XML cannot carry it.
|
||||
if !isXMLText(in.Prefix) {
|
||||
in.Prefix = urlPathEscape(in.Prefix)
|
||||
}
|
||||
in.Delimiter = urlPathEscape(in.Delimiter)
|
||||
}
|
||||
return listBucketResultV1{
|
||||
@@ -1615,11 +1618,6 @@ func (s3a *S3ApiServer) dirHoldsOnlyHiddenEntriesInner(ctx context.Context, clie
|
||||
func getListObjectsV2Args(values url.Values) (prefix, startAfter, delimiter string, token OptionalString, encodingTypeUrl bool, fetchOwner bool, maxkeys uint16, allowUnordered bool, errCode s3err.ErrorCode) {
|
||||
prefix = values.Get("prefix")
|
||||
token = OptionalString{set: values.Has("continuation-token"), string: values.Get("continuation-token")}
|
||||
// Every NextContinuationToken this server writes is non-empty.
|
||||
if token.set && token.string == "" {
|
||||
errCode = s3err.ErrInvalidContinuationToken
|
||||
return
|
||||
}
|
||||
startAfter = values.Get("start-after")
|
||||
delimiter = values.Get("delimiter")
|
||||
if encodingTypeUrl, errCode = parseEncodingType(values); errCode != s3err.ErrNone {
|
||||
|
||||
@@ -16,7 +16,7 @@ func TestGetListObjectsV2ArgsContinuationToken(t *testing.T) {
|
||||
want string
|
||||
wantErr s3err.ErrorCode
|
||||
}{
|
||||
{query: "continuation-token=", wantErr: s3err.ErrInvalidContinuationToken},
|
||||
{query: "continuation-token=&start-after=a", want: ""},
|
||||
{query: "continuation-token=photos%2F2006%2FJanuary%2Fsample.jpg", want: "photos/2006/January/sample.jpg"},
|
||||
{query: "continuation-token=" + encodeListContinuationToken("ctrl/\x01key"), want: "ctrl/\x01key"},
|
||||
{query: "continuation-token=base64%3Aplain", want: "base64:plain"},
|
||||
@@ -28,6 +28,7 @@ func TestGetListObjectsV2ArgsContinuationToken(t *testing.T) {
|
||||
_, _, _, token, _, _, _, _, errCode := getListObjectsV2Args(values)
|
||||
assert.Equal(t, tc.wantErr, errCode, tc.query)
|
||||
if tc.wantErr == s3err.ErrNone {
|
||||
assert.Equal(t, values.Has("continuation-token"), token.set, tc.query)
|
||||
assert.Equal(t, tc.want, decodeListContinuationToken(token.string), tc.query)
|
||||
}
|
||||
}
|
||||
@@ -69,10 +70,10 @@ func TestListObjectVersionsEncodeURL(t *testing.T) {
|
||||
Name: "b",
|
||||
Prefix: "dir a/",
|
||||
Delimiter: "/",
|
||||
KeyMarker: encodeListContinuationToken("dir a/café+1"),
|
||||
KeyMarker: encodeListContinuationToken("dir a/résumé+1"),
|
||||
NextKeyMarker: encodeListContinuationToken("dir a/100%"),
|
||||
Entries: []VersionListEntry{
|
||||
{Version: &VersionEntry{Key: "dir a/café+1"}},
|
||||
{Version: &VersionEntry{Key: "dir a/résumé+1"}},
|
||||
{DeleteMarker: &DeleteMarkerEntry{Key: "dir a/100%"}},
|
||||
{Prefix: &PrefixEntry{Prefix: "dir a/sub dir/"}},
|
||||
},
|
||||
@@ -85,27 +86,31 @@ func TestListObjectVersionsEncodeURL(t *testing.T) {
|
||||
for _, want := range []string{
|
||||
"<Prefix>dir%20a/</Prefix>",
|
||||
"<Delimiter>/</Delimiter>",
|
||||
"<KeyMarker>dir%20a/caf%C3%A9%2B1</KeyMarker>",
|
||||
"<KeyMarker>dir%20a/r%C3%A9sum%C3%A9%2B1</KeyMarker>",
|
||||
"<NextKeyMarker>dir%20a/100%25</NextKeyMarker>",
|
||||
"<Version><Key>dir%20a/caf%C3%A9%2B1</Key>",
|
||||
"<Version><Key>dir%20a/r%C3%A9sum%C3%A9%2B1</Key>",
|
||||
"<DeleteMarker><Key>dir%20a/100%25</Key>",
|
||||
"<CommonPrefixes><Prefix>dir%20a/sub%20dir/</Prefix></CommonPrefixes>",
|
||||
"<EncodingType>url</EncodingType>",
|
||||
} {
|
||||
assert.Contains(t, body, want)
|
||||
}
|
||||
decoded, err := url.PathUnescape("dir%20a/caf%C3%A9%2B1")
|
||||
decoded, err := url.PathUnescape("dir%20a/r%C3%A9sum%C3%A9%2B1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "dir a/café+1", decoded)
|
||||
assert.Equal(t, "dir a/résumé+1", decoded)
|
||||
}
|
||||
|
||||
func TestListObjectsV1EncodesPrefixAndDelimiter(t *testing.T) {
|
||||
response := ListBucketResult{Name: "b", Prefix: "dir a/café+", Delimiter: "/", MaxKeys: 1000, EncodingType: "url"}
|
||||
func TestListObjectsV1EncodesDelimiterNotPrefix(t *testing.T) {
|
||||
response := ListBucketResult{Name: "b", Prefix: "dir a/\n+", Delimiter: " ", MaxKeys: 1000, EncodingType: "url"}
|
||||
body := string(s3err.EncodeXMLResponse(toListBucketResultV1(response)))
|
||||
assert.Contains(t, body, "<Prefix>dir%20a/caf%C3%A9%2B</Prefix>")
|
||||
assert.Contains(t, body, "<Delimiter>/</Delimiter>")
|
||||
assert.Contains(t, body, "<Prefix>dir a/
+</Prefix>")
|
||||
assert.Contains(t, body, "<Delimiter>%20</Delimiter>")
|
||||
|
||||
response.Prefix = "dir a/\x01"
|
||||
body = string(s3err.EncodeXMLResponse(toListBucketResultV1(response)))
|
||||
assert.Contains(t, body, "<Prefix>dir%20a/%01</Prefix>")
|
||||
|
||||
response.EncodingType = ""
|
||||
body = string(s3err.EncodeXMLResponse(toListBucketResultV1(response)))
|
||||
assert.Contains(t, body, "<Prefix>dir a/café+</Prefix>")
|
||||
assert.Contains(t, body, "<Delimiter> </Delimiter>")
|
||||
}
|
||||
Reference in new issue
Block a user