mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
s3api: listing argument validation and encoding-type=url for versions, prefix and delimiter (#12753)
* s3api: reject an empty ListObjectsV2 continuation-token
AWS answers continuation-token= with 400 InvalidArgument ("The continuation
token provided is incorrect"); SeaweedFS listed from the start of the bucket.
No token this server issues is empty, and raw-key and base64: tokens keep
resuming as before.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK
* s3api: reject an encoding-type other than url on listings
AWS answers any encoding-type other than url with 400 InvalidArgument
("Invalid Encoding Method specified in Request") on ListObjects,
ListObjectsV2, ListObjectVersions, ListMultipartUploads and ListParts;
SeaweedFS ignored the value and listed unencoded keys.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK
* s3api: apply encoding-type=url to ListObjectVersions
The handler accepted encoding-type=url but returned raw keys. It now
URL-encodes Key in Version and DeleteMarker entries, CommonPrefixes, Prefix,
Delimiter, KeyMarker and NextKeyMarker with the urlPathEscape form ListObjects
uses, and echoes EncodingType.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK
* s3api: URL-encode Prefix and Delimiter in ListObjects responses
Under encoding-type=url, ListObjects and ListObjectsV2 encoded keys,
CommonPrefixes, markers and StartAfter but echoed Prefix and Delimiter raw,
so a client decoding them once corrupted a prefix holding '%' or '+'.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
646549eee1
commit
1fd5008d63
5 files changed
+181
-2
No files matched your search
@@ -69,6 +69,10 @@ type listBucketResultV1 struct {
|
||||
}
|
||||
|
||||
func toListBucketResultV1(in ListBucketResult) listBucketResultV1 {
|
||||
if in.EncodingType == s3.EncodingTypeUrl {
|
||||
in.Prefix = urlPathEscape(in.Prefix)
|
||||
in.Delimiter = urlPathEscape(in.Delimiter)
|
||||
}
|
||||
return listBucketResultV1{
|
||||
Metadata: in.Metadata,
|
||||
Name: in.Name,
|
||||
@@ -155,6 +159,8 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
|
||||
if encodingTypeUrl {
|
||||
responseV2.EncodingType = s3.EncodingTypeUrl
|
||||
responseV2.StartAfter = urlPathEscape(startAfter)
|
||||
responseV2.Prefix = urlPathEscape(response.Prefix)
|
||||
responseV2.Delimiter = urlPathEscape(response.Delimiter)
|
||||
}
|
||||
|
||||
glog.V(3).Infof("ListObjectsV2Handler response: %+v", responseV2)
|
||||
@@ -1609,9 +1615,16 @@ func (s3a *S3ApiServer) dirHoldsOnlyHiddenEntriesInner(ctx context.Context, clie
|
||||
func getListObjectsV2Args(values url.Values) (prefix, startAfter, delimiter string, token OptionalString, encodingTypeUrl bool, fetchOwner bool, maxkeys uint16, allowUnordered bool, errCode s3err.ErrorCode) {
|
||||
prefix = values.Get("prefix")
|
||||
token = OptionalString{set: values.Has("continuation-token"), string: values.Get("continuation-token")}
|
||||
// Every NextContinuationToken this server writes is non-empty.
|
||||
if token.set && token.string == "" {
|
||||
errCode = s3err.ErrInvalidContinuationToken
|
||||
return
|
||||
}
|
||||
startAfter = values.Get("start-after")
|
||||
delimiter = values.Get("delimiter")
|
||||
encodingTypeUrl = values.Get("encoding-type") == s3.EncodingTypeUrl
|
||||
if encodingTypeUrl, errCode = parseEncodingType(values); errCode != s3err.ErrNone {
|
||||
return
|
||||
}
|
||||
maxkeys, errCode = parseMaxKeys(values.Get("max-keys"))
|
||||
if errCode != s3err.ErrNone {
|
||||
return
|
||||
@@ -1626,7 +1639,9 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
|
||||
prefix = values.Get("prefix")
|
||||
marker = values.Get("marker")
|
||||
delimiter = values.Get("delimiter")
|
||||
encodingTypeUrl = values.Get("encoding-type") == "url"
|
||||
if encodingTypeUrl, errCode = parseEncodingType(values); errCode != s3err.ErrNone {
|
||||
return
|
||||
}
|
||||
maxkeys, errCode = parseMaxKeys(values.Get("max-keys"))
|
||||
if errCode != s3err.ErrNone {
|
||||
return
|
||||
@@ -1636,6 +1651,18 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
|
||||
return
|
||||
}
|
||||
|
||||
// parseEncodingType reports whether encoding-type=url was requested. AWS
|
||||
// accepts no other encoding and answers anything else with InvalidArgument.
|
||||
func parseEncodingType(values url.Values) (bool, s3err.ErrorCode) {
|
||||
switch values.Get("encoding-type") {
|
||||
case "":
|
||||
return false, s3err.ErrNone
|
||||
case s3.EncodingTypeUrl:
|
||||
return true, s3err.ErrNone
|
||||
}
|
||||
return false, s3err.ErrInvalidEncodingMethod
|
||||
}
|
||||
|
||||
// parseMaxKeys parses the max-keys query parameter. Like AWS S3, any integer in
|
||||
// [0, 2147483647] is accepted and the page size is capped at maxObjectListSizeLimit;
|
||||
// an empty value means the default limit, anything else is ErrInvalidMaxKeys.
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGetListObjectsV2ArgsContinuationToken(t *testing.T) {
|
||||
cases := []struct {
|
||||
query string
|
||||
want string
|
||||
wantErr s3err.ErrorCode
|
||||
}{
|
||||
{query: "continuation-token=", wantErr: s3err.ErrInvalidContinuationToken},
|
||||
{query: "continuation-token=photos%2F2006%2FJanuary%2Fsample.jpg", want: "photos/2006/January/sample.jpg"},
|
||||
{query: "continuation-token=" + encodeListContinuationToken("ctrl/\x01key"), want: "ctrl/\x01key"},
|
||||
{query: "continuation-token=base64%3Aplain", want: "base64:plain"},
|
||||
{query: "start-after=a"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
values, err := url.ParseQuery(tc.query)
|
||||
require.NoError(t, err)
|
||||
_, _, _, token, _, _, _, _, errCode := getListObjectsV2Args(values)
|
||||
assert.Equal(t, tc.wantErr, errCode, tc.query)
|
||||
if tc.wantErr == s3err.ErrNone {
|
||||
assert.Equal(t, tc.want, decodeListContinuationToken(token.string), tc.query)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListArgsEncodingType(t *testing.T) {
|
||||
cases := []struct {
|
||||
query string
|
||||
wantURL bool
|
||||
wantErr s3err.ErrorCode
|
||||
}{
|
||||
{query: ""},
|
||||
{query: "encoding-type=url", wantURL: true},
|
||||
{query: "encoding-type=URL", wantErr: s3err.ErrInvalidEncodingMethod},
|
||||
{query: "encoding-type=bogus", wantErr: s3err.ErrInvalidEncodingMethod},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
values, err := url.ParseQuery(tc.query)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, _, _, _, urlV2, _, _, _, errV2 := getListObjectsV2Args(values)
|
||||
assert.Equal(t, tc.wantErr, errV2, "v2 %q", tc.query)
|
||||
assert.Equal(t, tc.wantURL, urlV2, "v2 %q", tc.query)
|
||||
|
||||
_, _, _, urlV1, _, _, errV1 := getListObjectsV1Args(values)
|
||||
assert.Equal(t, tc.wantErr, errV1, "v1 %q", tc.query)
|
||||
assert.Equal(t, tc.wantURL, urlV1, "v1 %q", tc.query)
|
||||
|
||||
_, _, _, _, _, _, errUploads := getBucketMultipartResources(values)
|
||||
assert.Equal(t, tc.wantErr, errUploads, "uploads %q", tc.query)
|
||||
|
||||
_, _, _, _, errParts := getObjectResources(values)
|
||||
assert.Equal(t, tc.wantErr, errParts, "parts %q", tc.query)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListObjectVersionsEncodeURL(t *testing.T) {
|
||||
result := &S3ListObjectVersionsResult{
|
||||
Name: "b",
|
||||
Prefix: "dir a/",
|
||||
Delimiter: "/",
|
||||
KeyMarker: encodeListContinuationToken("dir a/café+1"),
|
||||
NextKeyMarker: encodeListContinuationToken("dir a/100%"),
|
||||
Entries: []VersionListEntry{
|
||||
{Version: &VersionEntry{Key: "dir a/café+1"}},
|
||||
{DeleteMarker: &DeleteMarkerEntry{Key: "dir a/100%"}},
|
||||
{Prefix: &PrefixEntry{Prefix: "dir a/sub dir/"}},
|
||||
},
|
||||
}
|
||||
result.encodeURL()
|
||||
|
||||
out, err := xml.Marshal(result)
|
||||
require.NoError(t, err)
|
||||
body := string(out)
|
||||
for _, want := range []string{
|
||||
"<Prefix>dir%20a/</Prefix>",
|
||||
"<Delimiter>/</Delimiter>",
|
||||
"<KeyMarker>dir%20a/caf%C3%A9%2B1</KeyMarker>",
|
||||
"<NextKeyMarker>dir%20a/100%25</NextKeyMarker>",
|
||||
"<Version><Key>dir%20a/caf%C3%A9%2B1</Key>",
|
||||
"<DeleteMarker><Key>dir%20a/100%25</Key>",
|
||||
"<CommonPrefixes><Prefix>dir%20a/sub%20dir/</Prefix></CommonPrefixes>",
|
||||
"<EncodingType>url</EncodingType>",
|
||||
} {
|
||||
assert.Contains(t, body, want)
|
||||
}
|
||||
decoded, err := url.PathUnescape("dir%20a/caf%C3%A9%2B1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "dir a/café+1", decoded)
|
||||
}
|
||||
|
||||
func TestListObjectsV1EncodesPrefixAndDelimiter(t *testing.T) {
|
||||
response := ListBucketResult{Name: "b", Prefix: "dir a/café+", Delimiter: "/", MaxKeys: 1000, EncodingType: "url"}
|
||||
body := string(s3err.EncodeXMLResponse(toListBucketResultV1(response)))
|
||||
assert.Contains(t, body, "<Prefix>dir%20a/caf%C3%A9%2B</Prefix>")
|
||||
assert.Contains(t, body, "<Delimiter>/</Delimiter>")
|
||||
|
||||
response.EncodingType = ""
|
||||
body = string(s3err.EncodeXMLResponse(toListBucketResultV1(response)))
|
||||
assert.Contains(t, body, "<Prefix>dir a/café+</Prefix>")
|
||||
}
|
||||
@@ -657,6 +657,9 @@ func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadID
|
||||
maxUploads = maxUploadsList
|
||||
}
|
||||
encodingType = values.Get("encoding-type")
|
||||
if _, encodingErr := parseEncodingType(values); encodingErr != s3err.ErrNone && errCode == s3err.ErrNone {
|
||||
errCode = encodingErr
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -680,6 +683,9 @@ func getObjectResources(values url.Values) (uploadID string, partNumberMarker, m
|
||||
maxParts = maxPartsList
|
||||
}
|
||||
encodingType = values.Get("encoding-type")
|
||||
if _, encodingErr := parseEncodingType(values); encodingErr != s3err.ErrNone && errCode == s3err.ErrNone {
|
||||
errCode = encodingErr
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
@@ -494,6 +495,26 @@ func (s3a *S3ApiServer) truncateAndSetMarkers(combinedList []versionListItem, ma
|
||||
return combinedList, nextKeyMarker, nextVersionIdMarker, isTruncated
|
||||
}
|
||||
|
||||
// encodeURL applies encoding-type=url to every key-bearing field, the form
|
||||
// ListObjects uses for its keys; clients decode them once.
|
||||
func (result *S3ListObjectVersionsResult) encodeURL() {
|
||||
result.EncodingType = s3.EncodingTypeUrl
|
||||
result.Prefix = urlPathEscape(result.Prefix)
|
||||
result.Delimiter = urlPathEscape(result.Delimiter)
|
||||
result.KeyMarker = urlPathEscape(result.KeyMarker)
|
||||
result.NextKeyMarker = urlPathEscape(result.NextKeyMarker)
|
||||
for _, e := range result.Entries {
|
||||
switch {
|
||||
case e.Version != nil:
|
||||
e.Version.Key = urlPathEscape(e.Version.Key)
|
||||
case e.DeleteMarker != nil:
|
||||
e.DeleteMarker.Key = urlPathEscape(e.DeleteMarker.Key)
|
||||
case e.Prefix != nil:
|
||||
e.Prefix.Prefix = urlPathEscape(e.Prefix.Prefix)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// splitIntoResult builds the final S3ListObjectVersionsResult from the combined list.
|
||||
// It populates a single Entries slice that preserves the interleaved sort order.
|
||||
func (s3a *S3ApiServer) splitIntoResult(combinedList []versionListItem, bucket, prefix, keyMarker, versionIdMarker, delimiter string, maxKeys int, isTruncated bool, nextKeyMarker, nextVersionIdMarker string) *S3ListObjectVersionsResult {
|
||||
@@ -1991,6 +2012,11 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http
|
||||
versionIdMarker := query.Get("version-id-marker")
|
||||
delimiter := query.Get("delimiter")
|
||||
|
||||
encodingTypeUrl, errCode := parseEncodingType(query)
|
||||
if errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
}
|
||||
maxKeys, errCode := parseMaxKeys(query.Get("max-keys"))
|
||||
if errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
@@ -2007,6 +2033,9 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http
|
||||
|
||||
// Set the original prefix in the response (not the normalized internal prefix)
|
||||
result.Prefix = originalPrefix
|
||||
if encodingTypeUrl {
|
||||
result.encodeURL()
|
||||
}
|
||||
|
||||
glog.V(3).Infof("ListObjectVersionsHandler response: %+v", result)
|
||||
writeSuccessResponseXML(w, r, result)
|
||||
|
||||
@@ -68,6 +68,7 @@ const (
|
||||
ErrInvalidMaxKeys
|
||||
ErrInvalidMaxBuckets
|
||||
ErrInvalidContinuationToken
|
||||
ErrInvalidEncodingMethod
|
||||
ErrInvalidMaxUploads
|
||||
ErrInvalidMaxParts
|
||||
ErrInvalidMaxDeleteObjects
|
||||
@@ -257,6 +258,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
|
||||
Description: "The continuation token provided is incorrect",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidEncodingMethod: {
|
||||
Code: "InvalidArgument",
|
||||
Description: "Invalid Encoding Method specified in Request",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidMaxParts: {
|
||||
Code: "InvalidArgument",
|
||||
Description: "Argument max-parts must be an integer between 0 and 2147483647",
|
||||
|
||||
Reference in new issue
Block a user