s3api: listing argument validation and encoding-type=url for versions, prefix and delimiter (#12753)

* s3api: reject an empty ListObjectsV2 continuation-token

AWS answers continuation-token= with 400 InvalidArgument ("The continuation
token provided is incorrect"); SeaweedFS listed from the start of the bucket.
No token this server issues is empty, and raw-key and base64: tokens keep
resuming as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK

* s3api: reject an encoding-type other than url on listings

AWS answers any encoding-type other than url with 400 InvalidArgument
("Invalid Encoding Method specified in Request") on ListObjects,
ListObjectsV2, ListObjectVersions, ListMultipartUploads and ListParts;
SeaweedFS ignored the value and listed unencoded keys.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK

* s3api: apply encoding-type=url to ListObjectVersions

The handler accepted encoding-type=url but returned raw keys. It now
URL-encodes Key in Version and DeleteMarker entries, CommonPrefixes, Prefix,
Delimiter, KeyMarker and NextKeyMarker with the urlPathEscape form ListObjects
uses, and echoes EncodingType.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK

* s3api: URL-encode Prefix and Delimiter in ListObjects responses

Under encoding-type=url, ListObjects and ListObjectsV2 encoded keys,
CommonPrefixes, markers and StartAfter but echoed Prefix and Delimiter raw,
so a client decoding them once corrupted a prefix holding '%' or '+'.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-18 15:55:41 -08:00
1 parent 646549eee1
commit 1fd5008d63
5 files changed
+181 -2

No files matched your search

+29 -2
View File
@@ -69,6 +69,10 @@ type listBucketResultV1 struct {
}
func toListBucketResultV1(in ListBucketResult) listBucketResultV1 {
if in.EncodingType == s3.EncodingTypeUrl {
in.Prefix = urlPathEscape(in.Prefix)
in.Delimiter = urlPathEscape(in.Delimiter)
}
return listBucketResultV1{
Metadata: in.Metadata,
Name: in.Name,
@@ -155,6 +159,8 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
if encodingTypeUrl {
responseV2.EncodingType = s3.EncodingTypeUrl
responseV2.StartAfter = urlPathEscape(startAfter)
responseV2.Prefix = urlPathEscape(response.Prefix)
responseV2.Delimiter = urlPathEscape(response.Delimiter)
}
glog.V(3).Infof("ListObjectsV2Handler response: %+v", responseV2)
@@ -1609,9 +1615,16 @@ func (s3a *S3ApiServer) dirHoldsOnlyHiddenEntriesInner(ctx context.Context, clie
func getListObjectsV2Args(values url.Values) (prefix, startAfter, delimiter string, token OptionalString, encodingTypeUrl bool, fetchOwner bool, maxkeys uint16, allowUnordered bool, errCode s3err.ErrorCode) {
prefix = values.Get("prefix")
token = OptionalString{set: values.Has("continuation-token"), string: values.Get("continuation-token")}
// Every NextContinuationToken this server writes is non-empty.
if token.set && token.string == "" {
errCode = s3err.ErrInvalidContinuationToken
return
}
startAfter = values.Get("start-after")
delimiter = values.Get("delimiter")
encodingTypeUrl = values.Get("encoding-type") == s3.EncodingTypeUrl
if encodingTypeUrl, errCode = parseEncodingType(values); errCode != s3err.ErrNone {
return
}
maxkeys, errCode = parseMaxKeys(values.Get("max-keys"))
if errCode != s3err.ErrNone {
return
@@ -1626,7 +1639,9 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
prefix = values.Get("prefix")
marker = values.Get("marker")
delimiter = values.Get("delimiter")
encodingTypeUrl = values.Get("encoding-type") == "url"
if encodingTypeUrl, errCode = parseEncodingType(values); errCode != s3err.ErrNone {
return
}
maxkeys, errCode = parseMaxKeys(values.Get("max-keys"))
if errCode != s3err.ErrNone {
return
@@ -1636,6 +1651,18 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
return
}
// parseEncodingType reports whether encoding-type=url was requested. AWS
// accepts no other encoding and answers anything else with InvalidArgument.
func parseEncodingType(values url.Values) (bool, s3err.ErrorCode) {
switch values.Get("encoding-type") {
case "":
return false, s3err.ErrNone
case s3.EncodingTypeUrl:
return true, s3err.ErrNone
}
return false, s3err.ErrInvalidEncodingMethod
}
// parseMaxKeys parses the max-keys query parameter. Like AWS S3, any integer in
// [0, 2147483647] is accepted and the page size is capped at maxObjectListSizeLimit;
// an empty value means the default limit, anything else is ErrInvalidMaxKeys.
@@ -0,0 +1,111 @@
package s3api
import (
"encoding/xml"
"net/url"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetListObjectsV2ArgsContinuationToken(t *testing.T) {
cases := []struct {
query string
want string
wantErr s3err.ErrorCode
}{
{query: "continuation-token=", wantErr: s3err.ErrInvalidContinuationToken},
{query: "continuation-token=photos%2F2006%2FJanuary%2Fsample.jpg", want: "photos/2006/January/sample.jpg"},
{query: "continuation-token=" + encodeListContinuationToken("ctrl/\x01key"), want: "ctrl/\x01key"},
{query: "continuation-token=base64%3Aplain", want: "base64:plain"},
{query: "start-after=a"},
}
for _, tc := range cases {
values, err := url.ParseQuery(tc.query)
require.NoError(t, err)
_, _, _, token, _, _, _, _, errCode := getListObjectsV2Args(values)
assert.Equal(t, tc.wantErr, errCode, tc.query)
if tc.wantErr == s3err.ErrNone {
assert.Equal(t, tc.want, decodeListContinuationToken(token.string), tc.query)
}
}
}
func TestListArgsEncodingType(t *testing.T) {
cases := []struct {
query string
wantURL bool
wantErr s3err.ErrorCode
}{
{query: ""},
{query: "encoding-type=url", wantURL: true},
{query: "encoding-type=URL", wantErr: s3err.ErrInvalidEncodingMethod},
{query: "encoding-type=bogus", wantErr: s3err.ErrInvalidEncodingMethod},
}
for _, tc := range cases {
values, err := url.ParseQuery(tc.query)
require.NoError(t, err)
_, _, _, _, urlV2, _, _, _, errV2 := getListObjectsV2Args(values)
assert.Equal(t, tc.wantErr, errV2, "v2 %q", tc.query)
assert.Equal(t, tc.wantURL, urlV2, "v2 %q", tc.query)
_, _, _, urlV1, _, _, errV1 := getListObjectsV1Args(values)
assert.Equal(t, tc.wantErr, errV1, "v1 %q", tc.query)
assert.Equal(t, tc.wantURL, urlV1, "v1 %q", tc.query)
_, _, _, _, _, _, errUploads := getBucketMultipartResources(values)
assert.Equal(t, tc.wantErr, errUploads, "uploads %q", tc.query)
_, _, _, _, errParts := getObjectResources(values)
assert.Equal(t, tc.wantErr, errParts, "parts %q", tc.query)
}
}
func TestListObjectVersionsEncodeURL(t *testing.T) {
result := &S3ListObjectVersionsResult{
Name: "b",
Prefix: "dir a/",
Delimiter: "/",
KeyMarker: encodeListContinuationToken("dir a/café+1"),
NextKeyMarker: encodeListContinuationToken("dir a/100%"),
Entries: []VersionListEntry{
{Version: &VersionEntry{Key: "dir a/café+1"}},
{DeleteMarker: &DeleteMarkerEntry{Key: "dir a/100%"}},
{Prefix: &PrefixEntry{Prefix: "dir a/sub dir/"}},
},
}
result.encodeURL()
out, err := xml.Marshal(result)
require.NoError(t, err)
body := string(out)
for _, want := range []string{
"<Prefix>dir%20a/</Prefix>",
"<Delimiter>/</Delimiter>",
"<KeyMarker>dir%20a/caf%C3%A9%2B1</KeyMarker>",
"<NextKeyMarker>dir%20a/100%25</NextKeyMarker>",
"<Version><Key>dir%20a/caf%C3%A9%2B1</Key>",
"<DeleteMarker><Key>dir%20a/100%25</Key>",
"<CommonPrefixes><Prefix>dir%20a/sub%20dir/</Prefix></CommonPrefixes>",
"<EncodingType>url</EncodingType>",
} {
assert.Contains(t, body, want)
}
decoded, err := url.PathUnescape("dir%20a/caf%C3%A9%2B1")
require.NoError(t, err)
assert.Equal(t, "dir a/café+1", decoded)
}
func TestListObjectsV1EncodesPrefixAndDelimiter(t *testing.T) {
response := ListBucketResult{Name: "b", Prefix: "dir a/café+", Delimiter: "/", MaxKeys: 1000, EncodingType: "url"}
body := string(s3err.EncodeXMLResponse(toListBucketResultV1(response)))
assert.Contains(t, body, "<Prefix>dir%20a/caf%C3%A9%2B</Prefix>")
assert.Contains(t, body, "<Delimiter>/</Delimiter>")
response.EncodingType = ""
body = string(s3err.EncodeXMLResponse(toListBucketResultV1(response)))
assert.Contains(t, body, "<Prefix>dir a/café+</Prefix>")
}
@@ -657,6 +657,9 @@ func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadID
maxUploads = maxUploadsList
}
encodingType = values.Get("encoding-type")
if _, encodingErr := parseEncodingType(values); encodingErr != s3err.ErrNone && errCode == s3err.ErrNone {
errCode = encodingErr
}
return
}
@@ -680,6 +683,9 @@ func getObjectResources(values url.Values) (uploadID string, partNumberMarker, m
maxParts = maxPartsList
}
encodingType = values.Get("encoding-type")
if _, encodingErr := parseEncodingType(values); encodingErr != s3err.ErrNone && errCode == s3err.ErrNone {
errCode = encodingErr
}
return
}
+29
View File
@@ -18,6 +18,7 @@ import (
"sync"
"time"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -494,6 +495,26 @@ func (s3a *S3ApiServer) truncateAndSetMarkers(combinedList []versionListItem, ma
return combinedList, nextKeyMarker, nextVersionIdMarker, isTruncated
}
// encodeURL applies encoding-type=url to every key-bearing field, the form
// ListObjects uses for its keys; clients decode them once.
func (result *S3ListObjectVersionsResult) encodeURL() {
result.EncodingType = s3.EncodingTypeUrl
result.Prefix = urlPathEscape(result.Prefix)
result.Delimiter = urlPathEscape(result.Delimiter)
result.KeyMarker = urlPathEscape(result.KeyMarker)
result.NextKeyMarker = urlPathEscape(result.NextKeyMarker)
for _, e := range result.Entries {
switch {
case e.Version != nil:
e.Version.Key = urlPathEscape(e.Version.Key)
case e.DeleteMarker != nil:
e.DeleteMarker.Key = urlPathEscape(e.DeleteMarker.Key)
case e.Prefix != nil:
e.Prefix.Prefix = urlPathEscape(e.Prefix.Prefix)
}
}
}
// splitIntoResult builds the final S3ListObjectVersionsResult from the combined list.
// It populates a single Entries slice that preserves the interleaved sort order.
func (s3a *S3ApiServer) splitIntoResult(combinedList []versionListItem, bucket, prefix, keyMarker, versionIdMarker, delimiter string, maxKeys int, isTruncated bool, nextKeyMarker, nextVersionIdMarker string) *S3ListObjectVersionsResult {
@@ -1991,6 +2012,11 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http
versionIdMarker := query.Get("version-id-marker")
delimiter := query.Get("delimiter")
encodingTypeUrl, errCode := parseEncodingType(query)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
maxKeys, errCode := parseMaxKeys(query.Get("max-keys"))
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
@@ -2007,6 +2033,9 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http
// Set the original prefix in the response (not the normalized internal prefix)
result.Prefix = originalPrefix
if encodingTypeUrl {
result.encodeURL()
}
glog.V(3).Infof("ListObjectVersionsHandler response: %+v", result)
writeSuccessResponseXML(w, r, result)
+6
View File
@@ -68,6 +68,7 @@ const (
ErrInvalidMaxKeys
ErrInvalidMaxBuckets
ErrInvalidContinuationToken
ErrInvalidEncodingMethod
ErrInvalidMaxUploads
ErrInvalidMaxParts
ErrInvalidMaxDeleteObjects
@@ -257,6 +258,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "The continuation token provided is incorrect",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidEncodingMethod: {
Code: "InvalidArgument",
Description: "Invalid Encoding Method specified in Request",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidMaxParts: {
Code: "InvalidArgument",
Description: "Argument max-parts must be an integer between 0 and 2147483647",