From ac7b4f4cf620a3477a58776124725636af9e02b6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2026 15:02:30 -0700 Subject: [PATCH] s3vectors: bound ListVectorBuckets and ListIndexes maxResults (#12218) * s3vectors: bound ListVectorBuckets and ListIndexes maxResults Both pass maxResults to the catalog unbounded; AWS caps these list pages at 100, the same way ListVectors is bounded and topK is bounded. * s3vectors: raise ListVectorBuckets/ListIndexes maxResults cap to 500 AWS documents a valid range of 1-500 for both list APIs; the shared limit of 100 rejected legitimate page sizes. * s3vectors: reject explicit maxResults of zero on list calls AWS requires maxResults >= 1, but a plain int field cannot tell an omitted field from an explicit zero. Making MaxResults a pointer keeps the catalog default for omitted fields while a supplied zero fails validation. * s3vectors: test max ListIndexes page against an existing bucket --- weed/s3api/s3vectors/handlers.go | 21 ++++++++++-- weed/s3api/s3vectors/handlers_test.go | 46 +++++++++++++++++++++++++++ weed/s3api/s3vectors/types.go | 4 +-- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index 19c4aa51df..160d7f9652 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -301,6 +301,14 @@ func (s *Server) handleListVectorBuckets(w http.ResponseWriter, r *http.Request) writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } + maxResults := 0 + if req.MaxResults != nil { + if *req.MaxResults < 1 || *req.MaxResults > vectorsMaxListPage { + writeError(w, http.StatusBadRequest, "ValidationException", fmt.Sprintf("maxResults must be between 1 and %d", vectorsMaxListPage)) + return + } + maxResults = *req.MaxResults + } var resp struct { TableBuckets []struct { Arn string `json:"arn"` @@ -315,7 +323,7 @@ func (s *Server) handleListVectorBuckets(w http.ResponseWriter, r *http.Request) // s3vectors actions, while showing ones they were not. err := s.internalCatalog(r.Context(), "ListTableBuckets", &s3tables.ListTableBucketsRequest{ - MaxBuckets: req.MaxResults, ContinuationToken: req.NextToken, Prefix: req.Prefix, + MaxBuckets: maxResults, ContinuationToken: req.NextToken, Prefix: req.Prefix, }, &resp) if err != nil { catalogErr(w, err) @@ -1085,6 +1093,14 @@ func (s *Server) handleListIndexes(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } + maxResults := 0 + if req.MaxResults != nil { + if *req.MaxResults < 1 || *req.MaxResults > vectorsMaxListPage { + writeError(w, http.StatusBadRequest, "ValidationException", fmt.Sprintf("maxResults must be between 1 and %d", vectorsMaxListPage)) + return + } + maxResults = *req.MaxResults + } if _, err := s.authorizeVector(r.Context(), r, "s3vectors:ListIndexes", arns.bucket, ""); err != nil { catalogErr(w, err) return @@ -1100,7 +1116,7 @@ func (s *Server) handleListIndexes(w http.ResponseWriter, r *http.Request) { err = s.internalCatalog(r.Context(), "ListTables", &s3tables.ListTablesRequest{ TableBucketARN: s.s3tablesArn(arns.bucket), Namespace: []string{indexNamespace}, - MaxTables: req.MaxResults, + MaxTables: maxResults, ContinuationToken: req.NextToken, Prefix: req.Prefix, }, &resp) @@ -1382,6 +1398,7 @@ const ( vectorsDefaultPage = 500 vectorsMaxPutBatch = 500 vectorsMaxPage = 1000 + vectorsMaxListPage = 500 vectorsMaxTopK = 10000 ) diff --git a/weed/s3api/s3vectors/handlers_test.go b/weed/s3api/s3vectors/handlers_test.go index f39c678876..47fed5f131 100644 --- a/weed/s3api/s3vectors/handlers_test.go +++ b/weed/s3api/s3vectors/handlers_test.go @@ -183,6 +183,52 @@ func TestListSegmentVectorsPageLimits(t *testing.T) { } } +// An explicit maxResults of zero is rejected like any out-of-range value; +// an omitted field still reaches the catalog as 0 and pages by default. +func TestListRequestMaxResultsBounds(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, newFakeVectors(0)) + + zero, over, max := 0, vectorsMaxListPage+1, vectorsMaxListPage + for name, req := range map[string]*ListVectorBucketsRequest{ + "zero": {MaxResults: &zero}, + "over": {MaxResults: &over}, + } { + if w := postJSONAs(t, "", s.handleListVectorBuckets, req); w.Code != http.StatusBadRequest { + t.Fatalf("ListVectorBuckets %s: %d %s, want 400", name, w.Code, w.Body.String()) + } + } + for name, req := range map[string]*ListIndexesRequest{ + "zero": {VectorBucketName: "vec-bucket", MaxResults: &zero}, + "over": {VectorBucketName: "vec-bucket", MaxResults: &over}, + } { + if w := postJSONAs(t, "", s.handleListIndexes, req); w.Code != http.StatusBadRequest { + t.Fatalf("ListIndexes %s: %d %s, want 400", name, w.Code, w.Body.String()) + } + } + if w := postJSONAs(t, "", s.handleListVectorBuckets, &ListVectorBucketsRequest{}); w.Code != http.StatusOK { + t.Fatalf("ListVectorBuckets omitted maxResults: %d %s", w.Code, w.Body.String()) + } + if w := postJSONAs(t, "", s.handleListVectorBuckets, &ListVectorBucketsRequest{MaxResults: &max}); w.Code != http.StatusOK { + t.Fatalf("ListVectorBuckets maxResults=%d: %d %s", max, w.Code, w.Body.String()) + } + if w := postJSONAs(t, "", s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + if w := postJSONAs(t, "", s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: "vec-bucket", + IndexName: "idx", + DataType: "float32", + Dimension: 3, + DistanceMetric: "euclidean", + }); w.Code != http.StatusOK { + t.Fatalf("create index: %d %s", w.Code, w.Body.String()) + } + if w := postJSONAs(t, "", s.handleListIndexes, &ListIndexesRequest{VectorBucketName: "vec-bucket", MaxResults: &max}); w.Code != http.StatusOK { + t.Fatalf("ListIndexes maxResults=%d: %d %s", max, w.Code, w.Body.String()) + } +} + func TestArnParts(t *testing.T) { var v vectorBucketARNs diff --git a/weed/s3api/s3vectors/types.go b/weed/s3api/s3vectors/types.go index 066208f47a..423dc71949 100644 --- a/weed/s3api/s3vectors/types.go +++ b/weed/s3api/s3vectors/types.go @@ -75,7 +75,7 @@ type VectorBucket struct { } type ListVectorBucketsRequest struct { - MaxResults int `json:"maxResults"` + MaxResults *int `json:"maxResults"` NextToken string `json:"nextToken"` Prefix string `json:"prefix"` } @@ -158,7 +158,7 @@ type GetIndexResponse struct { } type ListIndexesRequest struct { - MaxResults int `json:"maxResults"` + MaxResults *int `json:"maxResults"` NextToken string `json:"nextToken"` Prefix string `json:"prefix"` VectorBucketName string `json:"vectorBucketName"`