diff --git a/weed/s3api/s3api_object_handlers_list.go b/weed/s3api/s3api_object_handlers_list.go
index 0c10c1795d..e90228f52c 100644
--- a/weed/s3api/s3api_object_handlers_list.go
+++ b/weed/s3api/s3api_object_handlers_list.go
@@ -69,6 +69,10 @@ type listBucketResultV1 struct {
}
func toListBucketResultV1(in ListBucketResult) listBucketResultV1 {
+ if in.EncodingType == s3.EncodingTypeUrl {
+ in.Prefix = urlPathEscape(in.Prefix)
+ in.Delimiter = urlPathEscape(in.Delimiter)
+ }
return listBucketResultV1{
Metadata: in.Metadata,
Name: in.Name,
@@ -155,6 +159,8 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
if encodingTypeUrl {
responseV2.EncodingType = s3.EncodingTypeUrl
responseV2.StartAfter = urlPathEscape(startAfter)
+ responseV2.Prefix = urlPathEscape(response.Prefix)
+ responseV2.Delimiter = urlPathEscape(response.Delimiter)
}
glog.V(3).Infof("ListObjectsV2Handler response: %+v", responseV2)
@@ -1609,9 +1615,16 @@ func (s3a *S3ApiServer) dirHoldsOnlyHiddenEntriesInner(ctx context.Context, clie
func getListObjectsV2Args(values url.Values) (prefix, startAfter, delimiter string, token OptionalString, encodingTypeUrl bool, fetchOwner bool, maxkeys uint16, allowUnordered bool, errCode s3err.ErrorCode) {
prefix = values.Get("prefix")
token = OptionalString{set: values.Has("continuation-token"), string: values.Get("continuation-token")}
+ // Every NextContinuationToken this server writes is non-empty.
+ if token.set && token.string == "" {
+ errCode = s3err.ErrInvalidContinuationToken
+ return
+ }
startAfter = values.Get("start-after")
delimiter = values.Get("delimiter")
- encodingTypeUrl = values.Get("encoding-type") == s3.EncodingTypeUrl
+ if encodingTypeUrl, errCode = parseEncodingType(values); errCode != s3err.ErrNone {
+ return
+ }
maxkeys, errCode = parseMaxKeys(values.Get("max-keys"))
if errCode != s3err.ErrNone {
return
@@ -1626,7 +1639,9 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
prefix = values.Get("prefix")
marker = values.Get("marker")
delimiter = values.Get("delimiter")
- encodingTypeUrl = values.Get("encoding-type") == "url"
+ if encodingTypeUrl, errCode = parseEncodingType(values); errCode != s3err.ErrNone {
+ return
+ }
maxkeys, errCode = parseMaxKeys(values.Get("max-keys"))
if errCode != s3err.ErrNone {
return
@@ -1636,6 +1651,18 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
return
}
+// parseEncodingType reports whether encoding-type=url was requested. AWS
+// accepts no other encoding and answers anything else with InvalidArgument.
+func parseEncodingType(values url.Values) (bool, s3err.ErrorCode) {
+ switch values.Get("encoding-type") {
+ case "":
+ return false, s3err.ErrNone
+ case s3.EncodingTypeUrl:
+ return true, s3err.ErrNone
+ }
+ return false, s3err.ErrInvalidEncodingMethod
+}
+
// parseMaxKeys parses the max-keys query parameter. Like AWS S3, any integer in
// [0, 2147483647] is accepted and the page size is capped at maxObjectListSizeLimit;
// an empty value means the default limit, anything else is ErrInvalidMaxKeys.
diff --git a/weed/s3api/s3api_object_handlers_list_args_test.go b/weed/s3api/s3api_object_handlers_list_args_test.go
new file mode 100644
index 0000000000..f648a1fa71
--- /dev/null
+++ b/weed/s3api/s3api_object_handlers_list_args_test.go
@@ -0,0 +1,111 @@
+package s3api
+
+import (
+ "encoding/xml"
+ "net/url"
+ "testing"
+
+ "github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetListObjectsV2ArgsContinuationToken(t *testing.T) {
+ cases := []struct {
+ query string
+ want string
+ wantErr s3err.ErrorCode
+ }{
+ {query: "continuation-token=", wantErr: s3err.ErrInvalidContinuationToken},
+ {query: "continuation-token=photos%2F2006%2FJanuary%2Fsample.jpg", want: "photos/2006/January/sample.jpg"},
+ {query: "continuation-token=" + encodeListContinuationToken("ctrl/\x01key"), want: "ctrl/\x01key"},
+ {query: "continuation-token=base64%3Aplain", want: "base64:plain"},
+ {query: "start-after=a"},
+ }
+ for _, tc := range cases {
+ values, err := url.ParseQuery(tc.query)
+ require.NoError(t, err)
+ _, _, _, token, _, _, _, _, errCode := getListObjectsV2Args(values)
+ assert.Equal(t, tc.wantErr, errCode, tc.query)
+ if tc.wantErr == s3err.ErrNone {
+ assert.Equal(t, tc.want, decodeListContinuationToken(token.string), tc.query)
+ }
+ }
+}
+
+func TestListArgsEncodingType(t *testing.T) {
+ cases := []struct {
+ query string
+ wantURL bool
+ wantErr s3err.ErrorCode
+ }{
+ {query: ""},
+ {query: "encoding-type=url", wantURL: true},
+ {query: "encoding-type=URL", wantErr: s3err.ErrInvalidEncodingMethod},
+ {query: "encoding-type=bogus", wantErr: s3err.ErrInvalidEncodingMethod},
+ }
+ for _, tc := range cases {
+ values, err := url.ParseQuery(tc.query)
+ require.NoError(t, err)
+
+ _, _, _, _, urlV2, _, _, _, errV2 := getListObjectsV2Args(values)
+ assert.Equal(t, tc.wantErr, errV2, "v2 %q", tc.query)
+ assert.Equal(t, tc.wantURL, urlV2, "v2 %q", tc.query)
+
+ _, _, _, urlV1, _, _, errV1 := getListObjectsV1Args(values)
+ assert.Equal(t, tc.wantErr, errV1, "v1 %q", tc.query)
+ assert.Equal(t, tc.wantURL, urlV1, "v1 %q", tc.query)
+
+ _, _, _, _, _, _, errUploads := getBucketMultipartResources(values)
+ assert.Equal(t, tc.wantErr, errUploads, "uploads %q", tc.query)
+
+ _, _, _, _, errParts := getObjectResources(values)
+ assert.Equal(t, tc.wantErr, errParts, "parts %q", tc.query)
+ }
+}
+
+func TestListObjectVersionsEncodeURL(t *testing.T) {
+ result := &S3ListObjectVersionsResult{
+ Name: "b",
+ Prefix: "dir a/",
+ Delimiter: "/",
+ KeyMarker: encodeListContinuationToken("dir a/café+1"),
+ NextKeyMarker: encodeListContinuationToken("dir a/100%"),
+ Entries: []VersionListEntry{
+ {Version: &VersionEntry{Key: "dir a/café+1"}},
+ {DeleteMarker: &DeleteMarkerEntry{Key: "dir a/100%"}},
+ {Prefix: &PrefixEntry{Prefix: "dir a/sub dir/"}},
+ },
+ }
+ result.encodeURL()
+
+ out, err := xml.Marshal(result)
+ require.NoError(t, err)
+ body := string(out)
+ for _, want := range []string{
+ "dir%20a/",
+ "/",
+ "dir%20a/caf%C3%A9%2B1",
+ "dir%20a/100%25",
+ "dir%20a/caf%C3%A9%2B1",
+ "dir%20a/100%25",
+ "dir%20a/sub%20dir/",
+ "url",
+ } {
+ assert.Contains(t, body, want)
+ }
+ decoded, err := url.PathUnescape("dir%20a/caf%C3%A9%2B1")
+ require.NoError(t, err)
+ assert.Equal(t, "dir a/café+1", decoded)
+}
+
+func TestListObjectsV1EncodesPrefixAndDelimiter(t *testing.T) {
+ response := ListBucketResult{Name: "b", Prefix: "dir a/café+", Delimiter: "/", MaxKeys: 1000, EncodingType: "url"}
+ body := string(s3err.EncodeXMLResponse(toListBucketResultV1(response)))
+ assert.Contains(t, body, "dir%20a/caf%C3%A9%2B")
+ assert.Contains(t, body, "/")
+
+ response.EncodingType = ""
+ body = string(s3err.EncodeXMLResponse(toListBucketResultV1(response)))
+ assert.Contains(t, body, "dir a/café+")
+}
diff --git a/weed/s3api/s3api_object_handlers_multipart.go b/weed/s3api/s3api_object_handlers_multipart.go
index 1aa504704e..6a9458a4f6 100644
--- a/weed/s3api/s3api_object_handlers_multipart.go
+++ b/weed/s3api/s3api_object_handlers_multipart.go
@@ -657,6 +657,9 @@ func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadID
maxUploads = maxUploadsList
}
encodingType = values.Get("encoding-type")
+ if _, encodingErr := parseEncodingType(values); encodingErr != s3err.ErrNone && errCode == s3err.ErrNone {
+ errCode = encodingErr
+ }
return
}
@@ -680,6 +683,9 @@ func getObjectResources(values url.Values) (uploadID string, partNumberMarker, m
maxParts = maxPartsList
}
encodingType = values.Get("encoding-type")
+ if _, encodingErr := parseEncodingType(values); encodingErr != s3err.ErrNone && errCode == s3err.ErrNone {
+ errCode = encodingErr
+ }
return
}
diff --git a/weed/s3api/s3api_object_versioning.go b/weed/s3api/s3api_object_versioning.go
index d2bc38a515..ff1ea2d3fd 100644
--- a/weed/s3api/s3api_object_versioning.go
+++ b/weed/s3api/s3api_object_versioning.go
@@ -18,6 +18,7 @@ import (
"sync"
"time"
+ "github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -494,6 +495,26 @@ func (s3a *S3ApiServer) truncateAndSetMarkers(combinedList []versionListItem, ma
return combinedList, nextKeyMarker, nextVersionIdMarker, isTruncated
}
+// encodeURL applies encoding-type=url to every key-bearing field, the form
+// ListObjects uses for its keys; clients decode them once.
+func (result *S3ListObjectVersionsResult) encodeURL() {
+ result.EncodingType = s3.EncodingTypeUrl
+ result.Prefix = urlPathEscape(result.Prefix)
+ result.Delimiter = urlPathEscape(result.Delimiter)
+ result.KeyMarker = urlPathEscape(result.KeyMarker)
+ result.NextKeyMarker = urlPathEscape(result.NextKeyMarker)
+ for _, e := range result.Entries {
+ switch {
+ case e.Version != nil:
+ e.Version.Key = urlPathEscape(e.Version.Key)
+ case e.DeleteMarker != nil:
+ e.DeleteMarker.Key = urlPathEscape(e.DeleteMarker.Key)
+ case e.Prefix != nil:
+ e.Prefix.Prefix = urlPathEscape(e.Prefix.Prefix)
+ }
+ }
+}
+
// splitIntoResult builds the final S3ListObjectVersionsResult from the combined list.
// It populates a single Entries slice that preserves the interleaved sort order.
func (s3a *S3ApiServer) splitIntoResult(combinedList []versionListItem, bucket, prefix, keyMarker, versionIdMarker, delimiter string, maxKeys int, isTruncated bool, nextKeyMarker, nextVersionIdMarker string) *S3ListObjectVersionsResult {
@@ -1991,6 +2012,11 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http
versionIdMarker := query.Get("version-id-marker")
delimiter := query.Get("delimiter")
+ encodingTypeUrl, errCode := parseEncodingType(query)
+ if errCode != s3err.ErrNone {
+ s3err.WriteErrorResponse(w, r, errCode)
+ return
+ }
maxKeys, errCode := parseMaxKeys(query.Get("max-keys"))
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
@@ -2007,6 +2033,9 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http
// Set the original prefix in the response (not the normalized internal prefix)
result.Prefix = originalPrefix
+ if encodingTypeUrl {
+ result.encodeURL()
+ }
glog.V(3).Infof("ListObjectVersionsHandler response: %+v", result)
writeSuccessResponseXML(w, r, result)
diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go
index 67358067d0..6b8c4b74ae 100644
--- a/weed/s3api/s3err/s3api_errors.go
+++ b/weed/s3api/s3err/s3api_errors.go
@@ -68,6 +68,7 @@ const (
ErrInvalidMaxKeys
ErrInvalidMaxBuckets
ErrInvalidContinuationToken
+ ErrInvalidEncodingMethod
ErrInvalidMaxUploads
ErrInvalidMaxParts
ErrInvalidMaxDeleteObjects
@@ -257,6 +258,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "The continuation token provided is incorrect",
HTTPStatusCode: http.StatusBadRequest,
},
+ ErrInvalidEncodingMethod: {
+ Code: "InvalidArgument",
+ Description: "Invalid Encoding Method specified in Request",
+ HTTPStatusCode: http.StatusBadRequest,
+ },
ErrInvalidMaxParts: {
Code: "InvalidArgument",
Description: "Argument max-parts must be an integer between 0 and 2147483647",