From d431c1b35a57b32b83818aab29ca7bb1120d6448 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 2 Nov 2026 01:15:42 -0800 Subject: [PATCH] s3api: let the vectors route, not the credential scope, raise the body limit (#12326) SigV4 verification picked the 32 MiB S3 Vectors body limit from the credential scope, which is not verified until the body is hashed. A request claiming the s3vectors scope on any other route could make the gateway buffer 32 MiB before authentication instead of 10 MiB. The vectors handler now sets the limit on the request context, and every other route keeps 10 MiB. Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK Co-authored-by: Claude Opus 5.5 --- weed/s3api/auth_signature_v4.go | 5 +---- weed/s3api/auto_signature_v4_test.go | 25 ++++++++++++++++--------- weed/s3api/s3_constants/header.go | 15 +++++++++++++++ weed/s3api/s3vectors/auth_test.go | 11 ++++++++--- weed/s3api/s3vectors/server.go | 1 + 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 2332d36a3a..b86c14b42c 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -653,10 +653,7 @@ func extractV4AuthInfoFromHeader(r *http.Request) (*v4AuthInfo, s3err.ErrorCode) hashedPayload := getContentSha256Cksum(r) if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil { - sizeLimit := int64(iamRequestBodyLimit) - if signV4Values.Credential.scope.service == "s3vectors" { - sizeLimit = s3_constants.S3VectorsMaxRequestBodyBytes - } + sizeLimit := s3_constants.GetRequestBodyLimitFromContext(r, iamRequestBodyLimit) var hashErr error hashedPayload, hashErr = streamHashRequestBody(r, sizeLimit) if errors.Is(hashErr, errRequestBodyTooLarge) { diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index 34874ae8dc..3ba5b874e1 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -1930,7 +1930,7 @@ func TestStreamingWithSizeLimit(t *testing.T) { assert.ErrorIs(t, err, errRequestBodyTooLarge) } -func TestNonS3BodyHashLimitFollowsService(t *testing.T) { +func TestNonS3BodyHashLimitFollowsRoute(t *testing.T) { iam := &IdentityAccessManagement{} assert.NoError(t, iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{ Identities: []*iam_pb.Identity{{ @@ -1941,20 +1941,27 @@ func TestNonS3BodyHashLimitFollowsService(t *testing.T) { })) signer := v4.NewSigner(credentials.NewStaticCredentials("vec_key", "vec_secret", "")) + vectorsRoute := int64(s3_constants.S3VectorsMaxRequestBodyBytes) for _, tc := range []struct { - service string - size int - want s3err.ErrorCode + service string + routeLimit int64 + size int + want s3err.ErrorCode }{ - {"s3vectors", 20 << 20, s3err.ErrNone}, - {"s3vectors", s3_constants.S3VectorsMaxRequestBodyBytes, s3err.ErrNone}, - {"s3vectors", s3_constants.S3VectorsMaxRequestBodyBytes + 1, s3err.ErrEntityTooLarge}, - {"iam", iamRequestBodyLimit + 1, s3err.ErrEntityTooLarge}, + {"s3vectors", vectorsRoute, 20 << 20, s3err.ErrNone}, + {"s3vectors", vectorsRoute, s3_constants.S3VectorsMaxRequestBodyBytes, s3err.ErrNone}, + {"s3vectors", vectorsRoute, s3_constants.S3VectorsMaxRequestBodyBytes + 1, s3err.ErrEntityTooLarge}, + // An s3vectors scope alone, on another route, keeps the 10 MiB limit. + {"s3vectors", 0, iamRequestBodyLimit + 1, s3err.ErrEntityTooLarge}, + {"iam", 0, iamRequestBodyLimit + 1, s3err.ErrEntityTooLarge}, } { - t.Run(fmt.Sprintf("%s_%d", tc.service, tc.size), func(t *testing.T) { + t.Run(fmt.Sprintf("%s_route%d_%d", tc.service, tc.routeLimit, tc.size), func(t *testing.T) { body := strings.Repeat("v", tc.size) req, err := http.NewRequest("POST", "http://localhost:8333/PutVectors", strings.NewReader(body)) assert.NoError(t, err) + if tc.routeLimit > 0 { + req = req.WithContext(s3_constants.SetRequestBodyLimitInContext(req.Context(), tc.routeLimit)) + } _, err = signer.Sign(req, strings.NewReader(body), tc.service, "us-east-1", time.Now()) assert.NoError(t, err) diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index a49d40220e..6256d4376b 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -359,6 +359,7 @@ const ( contextKeyOriginalRequestPath contextKey = "s3-original-request-path" contextKeyAccessKeyId contextKey = "s3-access-key-id" contextKeyStoredAccessKeyId contextKey = "s3-stored-access-key-id" + contextKeyRequestBodyLimit contextKey = "s3-request-body-limit" ) // identityHolder is a mutable container for the authenticated identity name, @@ -529,6 +530,20 @@ func GetOriginalRequestPathFromContext(r *http.Request) string { return "" } +// SetRequestBodyLimitInContext sets how much of the body SigV4 verification +// reads. The route sets it: the credential scope is not verified yet. +func SetRequestBodyLimitInContext(ctx context.Context, limit int64) context.Context { + return context.WithValue(ctx, contextKeyRequestBodyLimit, limit) +} + +// GetRequestBodyLimitFromContext returns the route's limit, or def. +func GetRequestBodyLimitFromContext(r *http.Request, def int64) int64 { + if limit, ok := r.Context().Value(contextKeyRequestBodyLimit).(int64); ok { + return limit + } + return def +} + // SetAccessKeyIdInContext stores the access key id the request authenticated // with: a static key's id, or an STS session's temporary key id. Several keys // can belong to one identity, so the identity name alone cannot tell their diff --git a/weed/s3api/s3vectors/auth_test.go b/weed/s3api/s3vectors/auth_test.go index 56e326e5a9..a7e1fb88b7 100644 --- a/weed/s3api/s3vectors/auth_test.go +++ b/weed/s3api/s3vectors/auth_test.go @@ -14,13 +14,15 @@ import ( ) type fakeAuthenticator struct { - identity *s3api.Identity - errCode s3err.ErrorCode - calls int + identity *s3api.Identity + errCode s3err.ErrorCode + calls int + bodyLimit int64 } func (f *fakeAuthenticator) AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) { f.calls++ + f.bodyLimit = s3_constants.GetRequestBodyLimitFromContext(r, 0) if f.errCode != s3err.ErrNone { return "", nil, f.errCode } @@ -71,6 +73,9 @@ func TestAuthAnswersOversizedBodyWithValidationException(t *testing.T) { if rec.Code != tc.wantStatus || auth.calls != tc.wantAuthCalls { t.Fatalf("status %d after %d auth calls, want %d after %d: %s", rec.Code, auth.calls, tc.wantStatus, tc.wantAuthCalls, rec.Body) } + if auth.calls > 0 && auth.bodyLimit != maxRequestBodyBytes { + t.Fatalf("signature check reads up to %d bytes, want the route's %d", auth.bodyLimit, maxRequestBodyBytes) + } if tc.wantCode != "" && !strings.Contains(rec.Body.String(), `"code":"`+tc.wantCode+`"`) { t.Fatalf("body %s, want code %s", rec.Body, tc.wantCode) } diff --git a/weed/s3api/s3vectors/server.go b/weed/s3api/s3vectors/server.go index ceb7389643..c2bb46a607 100644 --- a/weed/s3api/s3vectors/server.go +++ b/weed/s3api/s3vectors/server.go @@ -133,6 +133,7 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { writeError(w, http.StatusBadRequest, "ValidationException", errRequestBodyTooLarge.Error()) return } + r = r.WithContext(s3_constants.SetRequestBodyLimitInContext(r.Context(), maxRequestBodyBytes)) identity, identityObj, errCode := s.authenticator.AuthenticateRequest(r) switch errCode { case s3err.ErrNone: