From fc9be77dbb952e94a744f11eb1881b4d2bf96548 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2026 11:22:48 -0700 Subject: [PATCH] s3vectors: bound QueryVectors topK at the AWS limit (#12202) * s3vectors: bound QueryVectors topK at the AWS limit of 30 topK passed through unbounded; AWS caps a query at 30 results, so a larger request is a ValidationException before it reaches the engine. * s3vectors: raise QueryVectors topK bound to the AWS limit of 10,000 * s3vectors: validate QueryVectors topK before the catalog lookup * s3vectors: cover the QueryVectors topK bounds --- weed/s3api/s3vectors/handlers.go | 14 ++++++---- weed/s3api/s3vectors/handlers_test.go | 40 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index 791d9ad983..1a77c21fa5 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -995,12 +995,13 @@ func (s *Server) handleListVectors(w http.ResponseWriter, r *http.Request) { }) } -// Mirrors the engine's page sizing so a segmented listing pages like a plain -// one does. +// The page constants mirror the engine's page sizing so a segmented listing +// pages like a plain one does; vectorsMaxTopK is the AWS QueryVectors limit. const ( vectorsDefaultPage = 500 vectorsMaxPutBatch = 500 vectorsMaxPage = 1000 + vectorsMaxTopK = 10000 ) // listSegmentVectors scans the index from offset, keeping the rows that fall @@ -1360,6 +1361,11 @@ func (s *Server) handleQueryVectors(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", "queryVector is required") return } + if req.TopK <= 0 || req.TopK > vectorsMaxTopK { + writeError(w, http.StatusBadRequest, "ValidationException", + fmt.Sprintf("topK must be between 1 and %d", vectorsMaxTopK)) + return + } record, err := s.indexRecord(r.Context(), r, arns.bucket, arns.index) if err != nil { catalogErr(w, err) @@ -1382,10 +1388,6 @@ func (s *Server) handleQueryVectors(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - if req.TopK <= 0 { - writeError(w, http.StatusBadRequest, "ValidationException", "topK must be positive") - return - } resp, err := s.vectors.QueryVectors(r.Context(), &vector_pb.QueryVectorsRequest{ Table: tableID(arns.bucket, arns.index), QueryVector: queryVector, diff --git a/weed/s3api/s3vectors/handlers_test.go b/weed/s3api/s3vectors/handlers_test.go index 214c8c9dce..5776251971 100644 --- a/weed/s3api/s3vectors/handlers_test.go +++ b/weed/s3api/s3vectors/handlers_test.go @@ -11,6 +11,7 @@ import ( "strings" "testing" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/vector_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" @@ -507,3 +508,42 @@ func TestPutVectorsUsesCatalogDimension(t *testing.T) { t.Fatalf("engine PutVectors calls = %d, want a second call of %d vectors", len(engine.puts), vectorsMaxPutBatch) } } + +// errFiler fails any catalog access, so a request that passes request-shape +// validation dies in the catalog rather than reaching the engine. +type errFiler struct{} + +func (errFiler) WithFilerClient(_ bool, _ func(filer_pb.SeaweedFilerClient) error) error { + return fmt.Errorf("no filer") +} + +// Rejected topK values fail before the catalog and the engine; valid ones get +// past the gate (here: into the catalog, which the errFiler fails). +func TestQueryVectorsTopKBounds(t *testing.T) { + s := &Server{ + vectors: newFakeVectors(0), + tables: s3tables.NewManager(), + filerClient: errFiler{}, + } + for _, tc := range []struct { + name string + topK int + rejected bool + }{ + {"zero", 0, true}, + {"negative", -3, true}, + {"over the limit", vectorsMaxTopK + 1, true}, + {"one", 1, false}, + {"at the limit", vectorsMaxTopK, false}, + } { + t.Run(tc.name, func(t *testing.T) { + body := fmt.Sprintf(`{"vectorBucketName":"b","indexName":"i","queryVector":{"float32":[0.1]},"topK":%d}`, tc.topK) + w := httptest.NewRecorder() + s.handleQueryVectors(w, httptest.NewRequest(http.MethodPost, "/QueryVectors", strings.NewReader(body))) + rejected := w.Code == http.StatusBadRequest && strings.Contains(w.Body.String(), "topK must be between") + if rejected != tc.rejected { + t.Fatalf("topK %d: rejected=%v (%d %s), want %v", tc.topK, rejected, w.Code, w.Body.String(), tc.rejected) + } + }) + } +}