s3vectors: bound QueryVectors topK at the AWS limit (#12202)

* s3vectors: bound QueryVectors topK at the AWS limit of 30

topK passed through unbounded; AWS caps a query at 30 results, so a
larger request is a ValidationException before it reaches the engine.

* s3vectors: raise QueryVectors topK bound to the AWS limit of 10,000

* s3vectors: validate QueryVectors topK before the catalog lookup

* s3vectors: cover the QueryVectors topK bounds
This commit is contained in:
Chris Lu authored and GitHub committed 2026-10-29 11:22:48 -07:00
1 parent eecc461b39
commit fc9be77dbb
2 files changed
+48 -6

No files matched your search

+8 -6
View File
@@ -995,12 +995,13 @@ func (s *Server) handleListVectors(w http.ResponseWriter, r *http.Request) {
})
}
// Mirrors the engine's page sizing so a segmented listing pages like a plain
// one does.
// The page constants mirror the engine's page sizing so a segmented listing
// pages like a plain one does; vectorsMaxTopK is the AWS QueryVectors limit.
const (
vectorsDefaultPage = 500
vectorsMaxPutBatch = 500
vectorsMaxPage = 1000
vectorsMaxTopK = 10000
)
// listSegmentVectors scans the index from offset, keeping the rows that fall
@@ -1360,6 +1361,11 @@ func (s *Server) handleQueryVectors(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", "queryVector is required")
return
}
if req.TopK <= 0 || req.TopK > vectorsMaxTopK {
writeError(w, http.StatusBadRequest, "ValidationException",
fmt.Sprintf("topK must be between 1 and %d", vectorsMaxTopK))
return
}
record, err := s.indexRecord(r.Context(), r, arns.bucket, arns.index)
if err != nil {
catalogErr(w, err)
@@ -1382,10 +1388,6 @@ func (s *Server) handleQueryVectors(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
if req.TopK <= 0 {
writeError(w, http.StatusBadRequest, "ValidationException", "topK must be positive")
return
}
resp, err := s.vectors.QueryVectors(r.Context(), &vector_pb.QueryVectorsRequest{
Table: tableID(arns.bucket, arns.index),
QueryVector: queryVector,
+40
View File
@@ -11,6 +11,7 @@ import (
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/vector_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
@@ -507,3 +508,42 @@ func TestPutVectorsUsesCatalogDimension(t *testing.T) {
t.Fatalf("engine PutVectors calls = %d, want a second call of %d vectors", len(engine.puts), vectorsMaxPutBatch)
}
}
// errFiler fails any catalog access, so a request that passes request-shape
// validation dies in the catalog rather than reaching the engine.
type errFiler struct{}
func (errFiler) WithFilerClient(_ bool, _ func(filer_pb.SeaweedFilerClient) error) error {
return fmt.Errorf("no filer")
}
// Rejected topK values fail before the catalog and the engine; valid ones get
// past the gate (here: into the catalog, which the errFiler fails).
func TestQueryVectorsTopKBounds(t *testing.T) {
s := &Server{
vectors: newFakeVectors(0),
tables: s3tables.NewManager(),
filerClient: errFiler{},
}
for _, tc := range []struct {
name string
topK int
rejected bool
}{
{"zero", 0, true},
{"negative", -3, true},
{"over the limit", vectorsMaxTopK + 1, true},
{"one", 1, false},
{"at the limit", vectorsMaxTopK, false},
} {
t.Run(tc.name, func(t *testing.T) {
body := fmt.Sprintf(`{"vectorBucketName":"b","indexName":"i","queryVector":{"float32":[0.1]},"topK":%d}`, tc.topK)
w := httptest.NewRecorder()
s.handleQueryVectors(w, httptest.NewRequest(http.MethodPost, "/QueryVectors", strings.NewReader(body)))
rejected := w.Code == http.StatusBadRequest && strings.Contains(w.Body.String(), "topK must be between")
if rejected != tc.rejected {
t.Fatalf("topK %d: rejected=%v (%d %s), want %v", tc.topK, rejected, w.Code, w.Body.String(), tc.rejected)
}
})
}
}