diff --git a/weed/s3api/s3api_object_handlers_list.go b/weed/s3api/s3api_object_handlers_list.go index e90228f52c..f0eb1ced07 100644 --- a/weed/s3api/s3api_object_handlers_list.go +++ b/weed/s3api/s3api_object_handlers_list.go @@ -70,7 +70,10 @@ type listBucketResultV1 struct { func toListBucketResultV1(in ListBucketResult) listBucketResultV1 { if in.EncodingType == s3.EncodingTypeUrl { - in.Prefix = urlPathEscape(in.Prefix) + // botocore does not decode the V1 Prefix, so it stays raw unless XML cannot carry it. + if !isXMLText(in.Prefix) { + in.Prefix = urlPathEscape(in.Prefix) + } in.Delimiter = urlPathEscape(in.Delimiter) } return listBucketResultV1{ @@ -1615,11 +1618,6 @@ func (s3a *S3ApiServer) dirHoldsOnlyHiddenEntriesInner(ctx context.Context, clie func getListObjectsV2Args(values url.Values) (prefix, startAfter, delimiter string, token OptionalString, encodingTypeUrl bool, fetchOwner bool, maxkeys uint16, allowUnordered bool, errCode s3err.ErrorCode) { prefix = values.Get("prefix") token = OptionalString{set: values.Has("continuation-token"), string: values.Get("continuation-token")} - // Every NextContinuationToken this server writes is non-empty. - if token.set && token.string == "" { - errCode = s3err.ErrInvalidContinuationToken - return - } startAfter = values.Get("start-after") delimiter = values.Get("delimiter") if encodingTypeUrl, errCode = parseEncodingType(values); errCode != s3err.ErrNone { diff --git a/weed/s3api/s3api_object_handlers_list_args_test.go b/weed/s3api/s3api_object_handlers_list_args_test.go index f648a1fa71..582230b263 100644 --- a/weed/s3api/s3api_object_handlers_list_args_test.go +++ b/weed/s3api/s3api_object_handlers_list_args_test.go @@ -16,7 +16,7 @@ func TestGetListObjectsV2ArgsContinuationToken(t *testing.T) { want string wantErr s3err.ErrorCode }{ - {query: "continuation-token=", wantErr: s3err.ErrInvalidContinuationToken}, + {query: "continuation-token=&start-after=a", want: ""}, {query: "continuation-token=photos%2F2006%2FJanuary%2Fsample.jpg", want: "photos/2006/January/sample.jpg"}, {query: "continuation-token=" + encodeListContinuationToken("ctrl/\x01key"), want: "ctrl/\x01key"}, {query: "continuation-token=base64%3Aplain", want: "base64:plain"}, @@ -28,6 +28,7 @@ func TestGetListObjectsV2ArgsContinuationToken(t *testing.T) { _, _, _, token, _, _, _, _, errCode := getListObjectsV2Args(values) assert.Equal(t, tc.wantErr, errCode, tc.query) if tc.wantErr == s3err.ErrNone { + assert.Equal(t, values.Has("continuation-token"), token.set, tc.query) assert.Equal(t, tc.want, decodeListContinuationToken(token.string), tc.query) } } @@ -69,10 +70,10 @@ func TestListObjectVersionsEncodeURL(t *testing.T) { Name: "b", Prefix: "dir a/", Delimiter: "/", - KeyMarker: encodeListContinuationToken("dir a/café+1"), + KeyMarker: encodeListContinuationToken("dir a/résumé+1"), NextKeyMarker: encodeListContinuationToken("dir a/100%"), Entries: []VersionListEntry{ - {Version: &VersionEntry{Key: "dir a/café+1"}}, + {Version: &VersionEntry{Key: "dir a/résumé+1"}}, {DeleteMarker: &DeleteMarkerEntry{Key: "dir a/100%"}}, {Prefix: &PrefixEntry{Prefix: "dir a/sub dir/"}}, }, @@ -85,27 +86,31 @@ func TestListObjectVersionsEncodeURL(t *testing.T) { for _, want := range []string{ "dir%20a/", "/", - "dir%20a/caf%C3%A9%2B1", + "dir%20a/r%C3%A9sum%C3%A9%2B1", "dir%20a/100%25", - "dir%20a/caf%C3%A9%2B1", + "dir%20a/r%C3%A9sum%C3%A9%2B1", "dir%20a/100%25", "dir%20a/sub%20dir/", "url", } { assert.Contains(t, body, want) } - decoded, err := url.PathUnescape("dir%20a/caf%C3%A9%2B1") + decoded, err := url.PathUnescape("dir%20a/r%C3%A9sum%C3%A9%2B1") require.NoError(t, err) - assert.Equal(t, "dir a/café+1", decoded) + assert.Equal(t, "dir a/résumé+1", decoded) } -func TestListObjectsV1EncodesPrefixAndDelimiter(t *testing.T) { - response := ListBucketResult{Name: "b", Prefix: "dir a/café+", Delimiter: "/", MaxKeys: 1000, EncodingType: "url"} +func TestListObjectsV1EncodesDelimiterNotPrefix(t *testing.T) { + response := ListBucketResult{Name: "b", Prefix: "dir a/\n+", Delimiter: " ", MaxKeys: 1000, EncodingType: "url"} body := string(s3err.EncodeXMLResponse(toListBucketResultV1(response))) - assert.Contains(t, body, "dir%20a/caf%C3%A9%2B") - assert.Contains(t, body, "/") + assert.Contains(t, body, "dir a/ +") + assert.Contains(t, body, "%20") + + response.Prefix = "dir a/\x01" + body = string(s3err.EncodeXMLResponse(toListBucketResultV1(response))) + assert.Contains(t, body, "dir%20a/%01") response.EncodingType = "" body = string(s3err.EncodeXMLResponse(toListBucketResultV1(response))) - assert.Contains(t, body, "dir a/café+") + assert.Contains(t, body, " ") }