s3vectors: enforce AWS metadata, key and filter limits (#12738)

* s3vectors: enforce AWS metadata, key and filter limits

PutVectors accepted vector metadata of any size and key count, bounded
only by the request body limit, and keys of any length; QueryVectors
accepted filters of any size. Apply the AWS S3 Vectors limits
(https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-vectors-limitations.html),
taking the ENHANCED index values where the modes differ, since this
implementation prefilters and takes no indexMode:

- 100 KB total metadata per vector, which also bounds a single
  non-filterable field;
- 2 KB filterable metadata per vector;
- 50 metadata keys per vector;
- 1024-character vector keys;
- at most 10 nonFilterableMetadataKeys of 1 to 63 characters at
  CreateIndex;
- 100 filter constraints per QueryVectors request.

A violation is a ValidationException before the engine is called.

* s3vectors: accept metadata whose JSON is exactly at the byte limits

metadataEntrySize counts a trailing comma for every member, so starting
from both braces overcounted by one and rejected metadata that encodes to
exactly 2 KB filterable or 100 KB total.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3vectors: rewrap the metadata limits header comment

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-11-18 13:49:15 -08:00
1 parent 3db54ee7e5
commit e037a9c1ce
4 files changed
+366 -4

No files matched your search

+18 -3
View File
@@ -12,6 +12,7 @@ import (
"strconv"
"strings"
"time"
"unicode/utf8"
awsarn "github.com/aws/aws-sdk-go-v2/aws/arn"
"github.com/google/uuid"
@@ -1134,6 +1135,10 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", "dimension must be 1..=4096")
return
}
if err := validateMetadataConfiguration(req.MetadataConfiguration); err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
if err := s3tables.ValidateTagsWithLimit(req.Tags, maxResourceTags); err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -1536,10 +1541,10 @@ func (s *Server) handlePutVectors(w http.ResponseWriter, r *http.Request) {
catalogErr(w, err)
return
}
dimension := indexFromRecord(s, arns.bucket, arns.index, record).Dimension
index := indexFromRecord(s, arns.bucket, arns.index, record)
vectors := make([]*vector_pb.Vector, 0, len(req.Vectors))
for _, v := range req.Vectors {
pbVector, err := toPutVector(v, dimension)
pbVector, err := toPutVector(v, index.Dimension, index.MetadataConfiguration)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -1557,13 +1562,19 @@ func (s *Server) handlePutVectors(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, &PutVectorsResponse{})
}
func toPutVector(v *PutInputVector, dimension int32) (*vector_pb.Vector, error) {
func toPutVector(v *PutInputVector, dimension int32, cfg *MetadataConfiguration) (*vector_pb.Vector, error) {
if v == nil {
return nil, fmt.Errorf("vectors contains a null element")
}
if v.Key == "" {
return nil, fmt.Errorf("vector key must not be empty")
}
if n := utf8.RuneCountInString(v.Key); n > vectorKeyMaxLength {
return nil, fmt.Errorf("vector key is %d characters, the maximum is %d", n, vectorKeyMaxLength)
}
if err := checkVectorMetadata(v.Key, v.Metadata, cfg); err != nil {
return nil, err
}
metadata, err := metadataStruct(v.Metadata)
if err != nil {
return nil, err
@@ -2169,6 +2180,10 @@ func (s *Server) handleQueryVectors(w http.ResponseWriter, r *http.Request) {
fmt.Sprintf("topK must be between 1 and %d", vectorsMaxTopK))
return
}
if err := checkFilterConstraints(req.Filter); err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
if _, err := s.authorizeVector(r.Context(), r, "s3vectors:QueryVectors", arns.bucket, arns.index); err != nil {
catalogErr(w, err)
return
+1 -1
View File
@@ -534,7 +534,7 @@ func TestToPutVectorDimensionCheck(t *testing.T) {
{"empty key", &PutInputVector{Data: &VectorData{Float32: []float32{1, 2, 3}}}, 3, false},
} {
t.Run(tc.name, func(t *testing.T) {
if _, err := toPutVector(tc.v, tc.dim); (err == nil) != tc.ok {
if _, err := toPutVector(tc.v, tc.dim, nil); (err == nil) != tc.ok {
t.Fatalf("toPutVector err = %v, want ok %v", err, tc.ok)
}
})
+134
View File
@@ -0,0 +1,134 @@
package s3vectors
import (
"encoding/json"
"fmt"
"slices"
"strings"
"unicode/utf8"
)
// AWS S3 Vectors limits on a vector's key and metadata, on an index's
// metadata configuration, and on a query's filter, from
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-vectors-limitations.html.
// Where AWS differs by index mode the ENHANCED value applies: this
// implementation prefilters, as ENHANCED indexes do, and takes no indexMode.
const (
// vectorKeyMaxLength is the PutInputVector key length limit.
vectorKeyMaxLength = 1024
// vectorMetadataMaxBytes is the total metadata size per vector, which
// also bounds a single non-filterable field (100 KB on ENHANCED indexes,
// 40 KB on CLASSIC).
vectorMetadataMaxBytes = 100 * 1024
// vectorFilterableMetadataMaxBytes is the filterable metadata size per vector.
vectorFilterableMetadataMaxBytes = 2 * 1024
// vectorMetadataMaxKeys is the number of metadata keys per vector.
vectorMetadataMaxKeys = 50
// indexNonFilterableMaxKeys and nonFilterableKeyMaxLength bound
// MetadataConfiguration.NonFilterableMetadataKeys at CreateIndex.
indexNonFilterableMaxKeys = 10
nonFilterableKeyMaxLength = 63
// queryFilterMaxConstraints is the number of filter constraints per
// QueryVectors request on an ENHANCED index.
queryFilterMaxConstraints = 100
)
// filterConstraints counts a filter's constraints: one per comparison on a
// field, so {"a": 1} is one and {"a": {"$gte": 1, "$lt": 5}} is two;
// $and, $or and their nesting add none of their own.
func filterConstraints(filter map[string]interface{}) int {
n := 0
for key, cond := range filter {
if strings.HasPrefix(key, "$") {
if sub, ok := cond.([]interface{}); ok {
for _, item := range sub {
if m, ok := item.(map[string]interface{}); ok {
n += filterConstraints(m)
}
}
}
continue
}
if ops, ok := cond.(map[string]interface{}); ok && len(ops) > 0 {
n += len(ops)
} else {
n++
}
}
return n
}
// checkFilterConstraints applies queryFilterMaxConstraints.
func checkFilterConstraints(filter map[string]interface{}) error {
if n := filterConstraints(filter); n > queryFilterMaxConstraints {
return fmt.Errorf("filter has %d constraints, the maximum is %d", n, queryFilterMaxConstraints)
}
return nil
}
// validateMetadataConfiguration applies the CreateIndex limits on the
// non-filterable metadata keys.
func validateMetadataConfiguration(cfg *MetadataConfiguration) error {
if cfg == nil {
return nil
}
keys := cfg.NonFilterableMetadataKeys
if len(keys) > indexNonFilterableMaxKeys {
return fmt.Errorf("nonFilterableMetadataKeys has %d keys, the maximum is %d", len(keys), indexNonFilterableMaxKeys)
}
for _, k := range keys {
if n := utf8.RuneCountInString(k); n < 1 || n > nonFilterableKeyMaxLength {
return fmt.Errorf("nonFilterableMetadataKeys entries must be 1 to %d characters", nonFilterableKeyMaxLength)
}
}
return nil
}
// checkVectorMetadata applies the per-vector metadata limits. Sizes are the
// JSON encoding of the metadata, the form the request carried it in; the
// filterable size counts only the keys the index does not declare
// non-filterable.
func checkVectorMetadata(key string, metadata map[string]interface{}, cfg *MetadataConfiguration) error {
if len(metadata) == 0 {
return nil
}
if len(metadata) > vectorMetadataMaxKeys {
return fmt.Errorf("vector %q has %d metadata keys, the maximum is %d", key, len(metadata), vectorMetadataMaxKeys)
}
var nonFilterable []string
if cfg != nil {
nonFilterable = cfg.NonFilterableMetadataKeys
}
total, filterable := 1, 1 // the braces, less the last member's comma
for k, v := range metadata {
size, err := metadataEntrySize(k, v)
if err != nil {
return fmt.Errorf("vector %q metadata %q: %v", key, k, err)
}
total += size
if !slices.Contains(nonFilterable, k) {
filterable += size
}
}
if total > vectorMetadataMaxBytes {
return fmt.Errorf("vector %q has %d bytes of metadata, the maximum is %d", key, total, vectorMetadataMaxBytes)
}
if filterable > vectorFilterableMetadataMaxBytes {
return fmt.Errorf("vector %q has %d bytes of filterable metadata, the maximum is %d; declare large keys in nonFilterableMetadataKeys",
key, filterable, vectorFilterableMetadataMaxBytes)
}
return nil
}
// metadataEntrySize is the encoded size of one `"key":value,` member.
func metadataEntrySize(k string, v interface{}) (int, error) {
encodedKey, err := json.Marshal(k)
if err != nil {
return 0, err
}
encodedValue, err := json.Marshal(v)
if err != nil {
return 0, err
}
return len(encodedKey) + 1 + len(encodedValue) + 1, nil
}
@@ -0,0 +1,213 @@
package s3vectors
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
)
func vectorWithMetadata(key string, metadata map[string]interface{}) *PutInputVector {
return &PutInputVector{Key: key, Data: &VectorData{Float32: []float32{1, 2, 3}}, Metadata: metadata}
}
func TestToPutVectorMetadataLimits(t *testing.T) {
cfg := &MetadataConfiguration{NonFilterableMetadataKeys: []string{"body"}}
manyKeys := func(n int) map[string]interface{} {
m := make(map[string]interface{}, n)
for i := range n {
m[fmt.Sprintf("k%d", i)] = i
}
return m
}
for _, tc := range []struct {
name string
v *PutInputVector
ok bool
}{
{"no metadata", vectorWithMetadata("a", nil), true},
{"key at limit", vectorWithMetadata(strings.Repeat("k", vectorKeyMaxLength), nil), true},
{"key over limit", vectorWithMetadata(strings.Repeat("k", vectorKeyMaxLength+1), nil), false},
{"multibyte key at limit", vectorWithMetadata(strings.Repeat("é", vectorKeyMaxLength), nil), true},
{"keys at limit", vectorWithMetadata("a", manyKeys(vectorMetadataMaxKeys)), true},
{"keys over limit", vectorWithMetadata("a", manyKeys(vectorMetadataMaxKeys+1)), false},
{"small filterable", vectorWithMetadata("a", map[string]interface{}{"genre": "jazz"}), true},
{"filterable over 2KB", vectorWithMetadata("a", map[string]interface{}{"genre": strings.Repeat("x", vectorFilterableMetadataMaxBytes)}), false},
{"non-filterable over 2KB", vectorWithMetadata("a", map[string]interface{}{"body": strings.Repeat("x", 30*1024)}), true},
{"non-filterable over the CLASSIC 40KB", vectorWithMetadata("a", map[string]interface{}{"body": strings.Repeat("x", 60*1024)}), true},
{"total over 100KB", vectorWithMetadata("a", map[string]interface{}{"body": strings.Repeat("x", vectorMetadataMaxBytes)}), false},
{"total over 100KB across keys", vectorWithMetadata("a", map[string]interface{}{
"body": strings.Repeat("x", 60*1024), "genre": "jazz", "k2": strings.Repeat("y", 1024),
"k3": strings.Repeat("z", 45*1024),
}), false},
{"nested filterable counted", vectorWithMetadata("a", map[string]interface{}{
"tags": []interface{}{strings.Repeat("x", 1500), strings.Repeat("y", 1500)},
}), false},
} {
t.Run(tc.name, func(t *testing.T) {
if _, err := toPutVector(tc.v, 3, cfg); (err == nil) != tc.ok {
t.Fatalf("toPutVector err = %v, want ok %v", err, tc.ok)
}
})
}
}
// The filterable limit applies to every key when the index declares no
// non-filterable keys.
func TestToPutVectorFilterableWithoutConfiguration(t *testing.T) {
v := vectorWithMetadata("a", map[string]interface{}{"body": strings.Repeat("x", 4096)})
if _, err := toPutVector(v, 3, nil); err == nil {
t.Fatal("4KB metadata on an index without non-filterable keys was accepted")
}
}
func TestValidateMetadataConfiguration(t *testing.T) {
keys := func(n int) []string {
out := make([]string, n)
for i := range out {
out[i] = fmt.Sprintf("k%d", i)
}
return out
}
for _, tc := range []struct {
name string
cfg *MetadataConfiguration
ok bool
}{
{"nil", nil, true},
{"ten keys", &MetadataConfiguration{NonFilterableMetadataKeys: keys(indexNonFilterableMaxKeys)}, true},
{"eleven keys", &MetadataConfiguration{NonFilterableMetadataKeys: keys(indexNonFilterableMaxKeys + 1)}, false},
{"empty key", &MetadataConfiguration{NonFilterableMetadataKeys: []string{""}}, false},
{"long key", &MetadataConfiguration{NonFilterableMetadataKeys: []string{strings.Repeat("k", nonFilterableKeyMaxLength+1)}}, false},
} {
t.Run(tc.name, func(t *testing.T) {
if err := validateMetadataConfiguration(tc.cfg); (err == nil) != tc.ok {
t.Fatalf("err = %v, want ok %v", err, tc.ok)
}
})
}
}
// End to end through the handlers: the index's own non-filterable keys
// decide what counts toward the filterable limit, and a vector over a limit
// is refused with a ValidationException before the engine sees the batch.
func TestPutVectorsEnforcesMetadataLimits(t *testing.T) {
fs := s3tablestest.Start(t)
engine := &fakeVectors{}
s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
if w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK {
t.Fatalf("create bucket: %d %s", w.Code, w.Body.String())
}
tooMany := make([]string, indexNonFilterableMaxKeys+1)
for i := range tooMany {
tooMany[i] = fmt.Sprintf("k%d", i)
}
if w := postJSON(t, s.handleCreateIndex, &CreateIndexRequest{
VectorBucketName: "vec-bucket", IndexName: "bad", DataType: "float32", Dimension: 3, DistanceMetric: "euclidean",
MetadataConfiguration: &MetadataConfiguration{NonFilterableMetadataKeys: tooMany},
}); w.Code != http.StatusBadRequest || !strings.Contains(w.Body.String(), "ValidationException") {
t.Fatalf("create index with %d non-filterable keys: %d %s", len(tooMany), w.Code, w.Body.String())
}
if len(engine.created) != 0 {
t.Fatalf("rejected index reached the engine: %v", engine.created)
}
if w := postJSON(t, s.handleCreateIndex, &CreateIndexRequest{
VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "euclidean",
MetadataConfiguration: &MetadataConfiguration{NonFilterableMetadataKeys: []string{"body"}},
}); w.Code != http.StatusOK {
t.Fatalf("create index: %d %s", w.Code, w.Body.String())
}
put := func(v *PutInputVector) *http.Response {
w := postJSON(t, s.handlePutVectors, &PutVectorsRequest{VectorBucketName: "vec-bucket", IndexName: "idx", Vectors: []*PutInputVector{v}})
return w.Result()
}
if resp := put(vectorWithMetadata("ok", map[string]interface{}{"body": strings.Repeat("x", 20*1024)})); resp.StatusCode != http.StatusOK {
t.Fatalf("large non-filterable metadata: %d", resp.StatusCode)
}
if resp := put(vectorWithMetadata("big", map[string]interface{}{"genre": strings.Repeat("x", 4096)})); resp.StatusCode != http.StatusBadRequest {
t.Fatalf("4KB filterable metadata: %d, want 400", resp.StatusCode)
}
if len(engine.puts) != 1 {
t.Fatalf("engine PutVectors calls = %d, want only the accepted vector", len(engine.puts))
}
}
func TestFilterConstraints(t *testing.T) {
for _, tc := range []struct {
name string
filter map[string]interface{}
want int
}{
{"none", nil, 0},
{"implicit equality", map[string]interface{}{"genre": "jazz"}, 1},
{"operator object", map[string]interface{}{"year": map[string]interface{}{"$gte": 1990, "$lt": 2000}}, 2},
{"nested logic", map[string]interface{}{"$and": []interface{}{
map[string]interface{}{"genre": "jazz"},
map[string]interface{}{"$or": []interface{}{
map[string]interface{}{"year": map[string]interface{}{"$eq": 1990}},
map[string]interface{}{"label": map[string]interface{}{"$in": []interface{}{"a", "b"}}},
}},
}}, 3},
} {
t.Run(tc.name, func(t *testing.T) {
if got := filterConstraints(tc.filter); got != tc.want {
t.Fatalf("filterConstraints = %d, want %d", got, tc.want)
}
})
}
}
func orFilter(n int) map[string]interface{} {
terms := make([]interface{}, n)
for i := range terms {
terms[i] = map[string]interface{}{fmt.Sprintf("k%d", i): i}
}
return map[string]interface{}{"$or": terms}
}
// A filter over the limit is refused before the catalog or the engine (the
// errFiler fails any catalog access); one at the limit gets past the gate.
func TestQueryVectorsFilterConstraintLimit(t *testing.T) {
s := &Server{
vectors: newFakeVectors(0),
tables: s3tables.NewManager(),
internal: s3tables.NewManager(),
filerClient: errFiler{},
}
query := func(filter map[string]interface{}) *httptest.ResponseRecorder {
return postJSON(t, s.handleQueryVectors, &QueryVectorsRequest{
VectorBucketName: "vec-bucket", IndexName: "idx", TopK: 1,
QueryVector: &QueryVector{Float32: []float32{1, 2, 3}}, Filter: filter,
})
}
w := query(orFilter(queryFilterMaxConstraints + 1))
if w.Code != http.StatusBadRequest || !strings.Contains(w.Body.String(), "ValidationException") ||
!strings.Contains(w.Body.String(), "constraints") {
t.Fatalf("%d constraints: %d %s, want 400 ValidationException", queryFilterMaxConstraints+1, w.Code, w.Body.String())
}
if w := query(orFilter(queryFilterMaxConstraints)); strings.Contains(w.Body.String(), "constraints") {
t.Fatalf("%d constraints refused: %d %s", queryFilterMaxConstraints, w.Code, w.Body.String())
}
}
func TestCheckVectorMetadataAcceptsExactLimits(t *testing.T) {
filterable := map[string]interface{}{"genre": strings.Repeat("x", vectorFilterableMetadataMaxBytes-12)}
if encoded, _ := json.Marshal(filterable); len(encoded) != vectorFilterableMetadataMaxBytes {
t.Fatalf("filterable metadata encodes to %d bytes, want %d", len(encoded), vectorFilterableMetadataMaxBytes)
}
if err := checkVectorMetadata("a", filterable, nil); err != nil {
t.Fatalf("filterable metadata of exactly %d bytes: %v", vectorFilterableMetadataMaxBytes, err)
}
total := map[string]interface{}{"genre": "x", "body": strings.Repeat("x", vectorMetadataMaxBytes-23)}
if encoded, _ := json.Marshal(total); len(encoded) != vectorMetadataMaxBytes {
t.Fatalf("metadata encodes to %d bytes, want %d", len(encoded), vectorMetadataMaxBytes)
}
if err := checkVectorMetadata("a", total, &MetadataConfiguration{NonFilterableMetadataKeys: []string{"body"}}); err != nil {
t.Fatalf("metadata of exactly %d bytes: %v", vectorMetadataMaxBytes, err)
}
}