From e037a9c1ce7d1f210fafd5da5dfe4775010f5729 Mon Sep 17 00:00:00 2001 From: MindAeon <30555587+mindaeon@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:49:15 +0300 Subject: [PATCH] s3vectors: enforce AWS metadata, key and filter limits (#12738) * s3vectors: enforce AWS metadata, key and filter limits PutVectors accepted vector metadata of any size and key count, bounded only by the request body limit, and keys of any length; QueryVectors accepted filters of any size. Apply the AWS S3 Vectors limits (https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-vectors-limitations.html), taking the ENHANCED index values where the modes differ, since this implementation prefilters and takes no indexMode: - 100 KB total metadata per vector, which also bounds a single non-filterable field; - 2 KB filterable metadata per vector; - 50 metadata keys per vector; - 1024-character vector keys; - at most 10 nonFilterableMetadataKeys of 1 to 63 characters at CreateIndex; - 100 filter constraints per QueryVectors request. A violation is a ValidationException before the engine is called. * s3vectors: accept metadata whose JSON is exactly at the byte limits metadataEntrySize counts a trailing comma for every member, so starting from both braces overcounted by one and rejected metadata that encodes to exactly 2 KB filterable or 100 KB total. Co-Authored-By: Claude Opus 5.5 * s3vectors: rewrap the metadata limits header comment Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Chris Lu Co-authored-by: Claude Opus 5.5 --- weed/s3api/s3vectors/handlers.go | 21 +- weed/s3api/s3vectors/handlers_test.go | 2 +- weed/s3api/s3vectors/metadata_limits.go | 134 ++++++++++++ weed/s3api/s3vectors/metadata_limits_test.go | 213 +++++++++++++++++++ 4 files changed, 366 insertions(+), 4 deletions(-) create mode 100644 weed/s3api/s3vectors/metadata_limits.go create mode 100644 weed/s3api/s3vectors/metadata_limits_test.go diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index e719a44883..933ce6d337 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "time" + "unicode/utf8" awsarn "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/google/uuid" @@ -1134,6 +1135,10 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", "dimension must be 1..=4096") return } + if err := validateMetadataConfiguration(req.MetadataConfiguration); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } if err := s3tables.ValidateTagsWithLimit(req.Tags, maxResourceTags); err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -1536,10 +1541,10 @@ func (s *Server) handlePutVectors(w http.ResponseWriter, r *http.Request) { catalogErr(w, err) return } - dimension := indexFromRecord(s, arns.bucket, arns.index, record).Dimension + index := indexFromRecord(s, arns.bucket, arns.index, record) vectors := make([]*vector_pb.Vector, 0, len(req.Vectors)) for _, v := range req.Vectors { - pbVector, err := toPutVector(v, dimension) + pbVector, err := toPutVector(v, index.Dimension, index.MetadataConfiguration) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -1557,13 +1562,19 @@ func (s *Server) handlePutVectors(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, &PutVectorsResponse{}) } -func toPutVector(v *PutInputVector, dimension int32) (*vector_pb.Vector, error) { +func toPutVector(v *PutInputVector, dimension int32, cfg *MetadataConfiguration) (*vector_pb.Vector, error) { if v == nil { return nil, fmt.Errorf("vectors contains a null element") } if v.Key == "" { return nil, fmt.Errorf("vector key must not be empty") } + if n := utf8.RuneCountInString(v.Key); n > vectorKeyMaxLength { + return nil, fmt.Errorf("vector key is %d characters, the maximum is %d", n, vectorKeyMaxLength) + } + if err := checkVectorMetadata(v.Key, v.Metadata, cfg); err != nil { + return nil, err + } metadata, err := metadataStruct(v.Metadata) if err != nil { return nil, err @@ -2169,6 +2180,10 @@ func (s *Server) handleQueryVectors(w http.ResponseWriter, r *http.Request) { fmt.Sprintf("topK must be between 1 and %d", vectorsMaxTopK)) return } + if err := checkFilterConstraints(req.Filter); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } if _, err := s.authorizeVector(r.Context(), r, "s3vectors:QueryVectors", arns.bucket, arns.index); err != nil { catalogErr(w, err) return diff --git a/weed/s3api/s3vectors/handlers_test.go b/weed/s3api/s3vectors/handlers_test.go index ed1914d09e..07436c468c 100644 --- a/weed/s3api/s3vectors/handlers_test.go +++ b/weed/s3api/s3vectors/handlers_test.go @@ -534,7 +534,7 @@ func TestToPutVectorDimensionCheck(t *testing.T) { {"empty key", &PutInputVector{Data: &VectorData{Float32: []float32{1, 2, 3}}}, 3, false}, } { t.Run(tc.name, func(t *testing.T) { - if _, err := toPutVector(tc.v, tc.dim); (err == nil) != tc.ok { + if _, err := toPutVector(tc.v, tc.dim, nil); (err == nil) != tc.ok { t.Fatalf("toPutVector err = %v, want ok %v", err, tc.ok) } }) diff --git a/weed/s3api/s3vectors/metadata_limits.go b/weed/s3api/s3vectors/metadata_limits.go new file mode 100644 index 0000000000..8a82151774 --- /dev/null +++ b/weed/s3api/s3vectors/metadata_limits.go @@ -0,0 +1,134 @@ +package s3vectors + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "unicode/utf8" +) + +// AWS S3 Vectors limits on a vector's key and metadata, on an index's +// metadata configuration, and on a query's filter, from +// https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-vectors-limitations.html. +// Where AWS differs by index mode the ENHANCED value applies: this +// implementation prefilters, as ENHANCED indexes do, and takes no indexMode. +const ( + // vectorKeyMaxLength is the PutInputVector key length limit. + vectorKeyMaxLength = 1024 + // vectorMetadataMaxBytes is the total metadata size per vector, which + // also bounds a single non-filterable field (100 KB on ENHANCED indexes, + // 40 KB on CLASSIC). + vectorMetadataMaxBytes = 100 * 1024 + // vectorFilterableMetadataMaxBytes is the filterable metadata size per vector. + vectorFilterableMetadataMaxBytes = 2 * 1024 + // vectorMetadataMaxKeys is the number of metadata keys per vector. + vectorMetadataMaxKeys = 50 + // indexNonFilterableMaxKeys and nonFilterableKeyMaxLength bound + // MetadataConfiguration.NonFilterableMetadataKeys at CreateIndex. + indexNonFilterableMaxKeys = 10 + nonFilterableKeyMaxLength = 63 + // queryFilterMaxConstraints is the number of filter constraints per + // QueryVectors request on an ENHANCED index. + queryFilterMaxConstraints = 100 +) + +// filterConstraints counts a filter's constraints: one per comparison on a +// field, so {"a": 1} is one and {"a": {"$gte": 1, "$lt": 5}} is two; +// $and, $or and their nesting add none of their own. +func filterConstraints(filter map[string]interface{}) int { + n := 0 + for key, cond := range filter { + if strings.HasPrefix(key, "$") { + if sub, ok := cond.([]interface{}); ok { + for _, item := range sub { + if m, ok := item.(map[string]interface{}); ok { + n += filterConstraints(m) + } + } + } + continue + } + if ops, ok := cond.(map[string]interface{}); ok && len(ops) > 0 { + n += len(ops) + } else { + n++ + } + } + return n +} + +// checkFilterConstraints applies queryFilterMaxConstraints. +func checkFilterConstraints(filter map[string]interface{}) error { + if n := filterConstraints(filter); n > queryFilterMaxConstraints { + return fmt.Errorf("filter has %d constraints, the maximum is %d", n, queryFilterMaxConstraints) + } + return nil +} + +// validateMetadataConfiguration applies the CreateIndex limits on the +// non-filterable metadata keys. +func validateMetadataConfiguration(cfg *MetadataConfiguration) error { + if cfg == nil { + return nil + } + keys := cfg.NonFilterableMetadataKeys + if len(keys) > indexNonFilterableMaxKeys { + return fmt.Errorf("nonFilterableMetadataKeys has %d keys, the maximum is %d", len(keys), indexNonFilterableMaxKeys) + } + for _, k := range keys { + if n := utf8.RuneCountInString(k); n < 1 || n > nonFilterableKeyMaxLength { + return fmt.Errorf("nonFilterableMetadataKeys entries must be 1 to %d characters", nonFilterableKeyMaxLength) + } + } + return nil +} + +// checkVectorMetadata applies the per-vector metadata limits. Sizes are the +// JSON encoding of the metadata, the form the request carried it in; the +// filterable size counts only the keys the index does not declare +// non-filterable. +func checkVectorMetadata(key string, metadata map[string]interface{}, cfg *MetadataConfiguration) error { + if len(metadata) == 0 { + return nil + } + if len(metadata) > vectorMetadataMaxKeys { + return fmt.Errorf("vector %q has %d metadata keys, the maximum is %d", key, len(metadata), vectorMetadataMaxKeys) + } + var nonFilterable []string + if cfg != nil { + nonFilterable = cfg.NonFilterableMetadataKeys + } + total, filterable := 1, 1 // the braces, less the last member's comma + for k, v := range metadata { + size, err := metadataEntrySize(k, v) + if err != nil { + return fmt.Errorf("vector %q metadata %q: %v", key, k, err) + } + total += size + if !slices.Contains(nonFilterable, k) { + filterable += size + } + } + if total > vectorMetadataMaxBytes { + return fmt.Errorf("vector %q has %d bytes of metadata, the maximum is %d", key, total, vectorMetadataMaxBytes) + } + if filterable > vectorFilterableMetadataMaxBytes { + return fmt.Errorf("vector %q has %d bytes of filterable metadata, the maximum is %d; declare large keys in nonFilterableMetadataKeys", + key, filterable, vectorFilterableMetadataMaxBytes) + } + return nil +} + +// metadataEntrySize is the encoded size of one `"key":value,` member. +func metadataEntrySize(k string, v interface{}) (int, error) { + encodedKey, err := json.Marshal(k) + if err != nil { + return 0, err + } + encodedValue, err := json.Marshal(v) + if err != nil { + return 0, err + } + return len(encodedKey) + 1 + len(encodedValue) + 1, nil +} diff --git a/weed/s3api/s3vectors/metadata_limits_test.go b/weed/s3api/s3vectors/metadata_limits_test.go new file mode 100644 index 0000000000..d8741373a2 --- /dev/null +++ b/weed/s3api/s3vectors/metadata_limits_test.go @@ -0,0 +1,213 @@ +package s3vectors + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" +) + +func vectorWithMetadata(key string, metadata map[string]interface{}) *PutInputVector { + return &PutInputVector{Key: key, Data: &VectorData{Float32: []float32{1, 2, 3}}, Metadata: metadata} +} + +func TestToPutVectorMetadataLimits(t *testing.T) { + cfg := &MetadataConfiguration{NonFilterableMetadataKeys: []string{"body"}} + manyKeys := func(n int) map[string]interface{} { + m := make(map[string]interface{}, n) + for i := range n { + m[fmt.Sprintf("k%d", i)] = i + } + return m + } + for _, tc := range []struct { + name string + v *PutInputVector + ok bool + }{ + {"no metadata", vectorWithMetadata("a", nil), true}, + {"key at limit", vectorWithMetadata(strings.Repeat("k", vectorKeyMaxLength), nil), true}, + {"key over limit", vectorWithMetadata(strings.Repeat("k", vectorKeyMaxLength+1), nil), false}, + {"multibyte key at limit", vectorWithMetadata(strings.Repeat("é", vectorKeyMaxLength), nil), true}, + {"keys at limit", vectorWithMetadata("a", manyKeys(vectorMetadataMaxKeys)), true}, + {"keys over limit", vectorWithMetadata("a", manyKeys(vectorMetadataMaxKeys+1)), false}, + {"small filterable", vectorWithMetadata("a", map[string]interface{}{"genre": "jazz"}), true}, + {"filterable over 2KB", vectorWithMetadata("a", map[string]interface{}{"genre": strings.Repeat("x", vectorFilterableMetadataMaxBytes)}), false}, + {"non-filterable over 2KB", vectorWithMetadata("a", map[string]interface{}{"body": strings.Repeat("x", 30*1024)}), true}, + {"non-filterable over the CLASSIC 40KB", vectorWithMetadata("a", map[string]interface{}{"body": strings.Repeat("x", 60*1024)}), true}, + {"total over 100KB", vectorWithMetadata("a", map[string]interface{}{"body": strings.Repeat("x", vectorMetadataMaxBytes)}), false}, + {"total over 100KB across keys", vectorWithMetadata("a", map[string]interface{}{ + "body": strings.Repeat("x", 60*1024), "genre": "jazz", "k2": strings.Repeat("y", 1024), + "k3": strings.Repeat("z", 45*1024), + }), false}, + {"nested filterable counted", vectorWithMetadata("a", map[string]interface{}{ + "tags": []interface{}{strings.Repeat("x", 1500), strings.Repeat("y", 1500)}, + }), false}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := toPutVector(tc.v, 3, cfg); (err == nil) != tc.ok { + t.Fatalf("toPutVector err = %v, want ok %v", err, tc.ok) + } + }) + } +} + +// The filterable limit applies to every key when the index declares no +// non-filterable keys. +func TestToPutVectorFilterableWithoutConfiguration(t *testing.T) { + v := vectorWithMetadata("a", map[string]interface{}{"body": strings.Repeat("x", 4096)}) + if _, err := toPutVector(v, 3, nil); err == nil { + t.Fatal("4KB metadata on an index without non-filterable keys was accepted") + } +} + +func TestValidateMetadataConfiguration(t *testing.T) { + keys := func(n int) []string { + out := make([]string, n) + for i := range out { + out[i] = fmt.Sprintf("k%d", i) + } + return out + } + for _, tc := range []struct { + name string + cfg *MetadataConfiguration + ok bool + }{ + {"nil", nil, true}, + {"ten keys", &MetadataConfiguration{NonFilterableMetadataKeys: keys(indexNonFilterableMaxKeys)}, true}, + {"eleven keys", &MetadataConfiguration{NonFilterableMetadataKeys: keys(indexNonFilterableMaxKeys + 1)}, false}, + {"empty key", &MetadataConfiguration{NonFilterableMetadataKeys: []string{""}}, false}, + {"long key", &MetadataConfiguration{NonFilterableMetadataKeys: []string{strings.Repeat("k", nonFilterableKeyMaxLength+1)}}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := validateMetadataConfiguration(tc.cfg); (err == nil) != tc.ok { + t.Fatalf("err = %v, want ok %v", err, tc.ok) + } + }) + } +} + +// End to end through the handlers: the index's own non-filterable keys +// decide what counts toward the filterable limit, and a vector over a limit +// is refused with a ValidationException before the engine sees the batch. +func TestPutVectorsEnforcesMetadataLimits(t *testing.T) { + fs := s3tablestest.Start(t) + engine := &fakeVectors{} + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine) + if w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + tooMany := make([]string, indexNonFilterableMaxKeys+1) + for i := range tooMany { + tooMany[i] = fmt.Sprintf("k%d", i) + } + if w := postJSON(t, s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: "vec-bucket", IndexName: "bad", DataType: "float32", Dimension: 3, DistanceMetric: "euclidean", + MetadataConfiguration: &MetadataConfiguration{NonFilterableMetadataKeys: tooMany}, + }); w.Code != http.StatusBadRequest || !strings.Contains(w.Body.String(), "ValidationException") { + t.Fatalf("create index with %d non-filterable keys: %d %s", len(tooMany), w.Code, w.Body.String()) + } + if len(engine.created) != 0 { + t.Fatalf("rejected index reached the engine: %v", engine.created) + } + if w := postJSON(t, s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "euclidean", + MetadataConfiguration: &MetadataConfiguration{NonFilterableMetadataKeys: []string{"body"}}, + }); w.Code != http.StatusOK { + t.Fatalf("create index: %d %s", w.Code, w.Body.String()) + } + put := func(v *PutInputVector) *http.Response { + w := postJSON(t, s.handlePutVectors, &PutVectorsRequest{VectorBucketName: "vec-bucket", IndexName: "idx", Vectors: []*PutInputVector{v}}) + return w.Result() + } + if resp := put(vectorWithMetadata("ok", map[string]interface{}{"body": strings.Repeat("x", 20*1024)})); resp.StatusCode != http.StatusOK { + t.Fatalf("large non-filterable metadata: %d", resp.StatusCode) + } + if resp := put(vectorWithMetadata("big", map[string]interface{}{"genre": strings.Repeat("x", 4096)})); resp.StatusCode != http.StatusBadRequest { + t.Fatalf("4KB filterable metadata: %d, want 400", resp.StatusCode) + } + if len(engine.puts) != 1 { + t.Fatalf("engine PutVectors calls = %d, want only the accepted vector", len(engine.puts)) + } +} + +func TestFilterConstraints(t *testing.T) { + for _, tc := range []struct { + name string + filter map[string]interface{} + want int + }{ + {"none", nil, 0}, + {"implicit equality", map[string]interface{}{"genre": "jazz"}, 1}, + {"operator object", map[string]interface{}{"year": map[string]interface{}{"$gte": 1990, "$lt": 2000}}, 2}, + {"nested logic", map[string]interface{}{"$and": []interface{}{ + map[string]interface{}{"genre": "jazz"}, + map[string]interface{}{"$or": []interface{}{ + map[string]interface{}{"year": map[string]interface{}{"$eq": 1990}}, + map[string]interface{}{"label": map[string]interface{}{"$in": []interface{}{"a", "b"}}}, + }}, + }}, 3}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := filterConstraints(tc.filter); got != tc.want { + t.Fatalf("filterConstraints = %d, want %d", got, tc.want) + } + }) + } +} + +func orFilter(n int) map[string]interface{} { + terms := make([]interface{}, n) + for i := range terms { + terms[i] = map[string]interface{}{fmt.Sprintf("k%d", i): i} + } + return map[string]interface{}{"$or": terms} +} + +// A filter over the limit is refused before the catalog or the engine (the +// errFiler fails any catalog access); one at the limit gets past the gate. +func TestQueryVectorsFilterConstraintLimit(t *testing.T) { + s := &Server{ + vectors: newFakeVectors(0), + tables: s3tables.NewManager(), + internal: s3tables.NewManager(), + filerClient: errFiler{}, + } + query := func(filter map[string]interface{}) *httptest.ResponseRecorder { + return postJSON(t, s.handleQueryVectors, &QueryVectorsRequest{ + VectorBucketName: "vec-bucket", IndexName: "idx", TopK: 1, + QueryVector: &QueryVector{Float32: []float32{1, 2, 3}}, Filter: filter, + }) + } + w := query(orFilter(queryFilterMaxConstraints + 1)) + if w.Code != http.StatusBadRequest || !strings.Contains(w.Body.String(), "ValidationException") || + !strings.Contains(w.Body.String(), "constraints") { + t.Fatalf("%d constraints: %d %s, want 400 ValidationException", queryFilterMaxConstraints+1, w.Code, w.Body.String()) + } + if w := query(orFilter(queryFilterMaxConstraints)); strings.Contains(w.Body.String(), "constraints") { + t.Fatalf("%d constraints refused: %d %s", queryFilterMaxConstraints, w.Code, w.Body.String()) + } +} + +func TestCheckVectorMetadataAcceptsExactLimits(t *testing.T) { + filterable := map[string]interface{}{"genre": strings.Repeat("x", vectorFilterableMetadataMaxBytes-12)} + if encoded, _ := json.Marshal(filterable); len(encoded) != vectorFilterableMetadataMaxBytes { + t.Fatalf("filterable metadata encodes to %d bytes, want %d", len(encoded), vectorFilterableMetadataMaxBytes) + } + if err := checkVectorMetadata("a", filterable, nil); err != nil { + t.Fatalf("filterable metadata of exactly %d bytes: %v", vectorFilterableMetadataMaxBytes, err) + } + total := map[string]interface{}{"genre": "x", "body": strings.Repeat("x", vectorMetadataMaxBytes-23)} + if encoded, _ := json.Marshal(total); len(encoded) != vectorMetadataMaxBytes { + t.Fatalf("metadata encodes to %d bytes, want %d", len(encoded), vectorMetadataMaxBytes) + } + if err := checkVectorMetadata("a", total, &MetadataConfiguration{NonFilterableMetadataKeys: []string{"body"}}); err != nil { + t.Fatalf("metadata of exactly %d bytes: %v", vectorMetadataMaxBytes, err) + } +}