mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
* s3vectors: enforce AWS metadata, key and filter limits PutVectors accepted vector metadata of any size and key count, bounded only by the request body limit, and keys of any length; QueryVectors accepted filters of any size. Apply the AWS S3 Vectors limits (https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-vectors-limitations.html), taking the ENHANCED index values where the modes differ, since this implementation prefilters and takes no indexMode: - 100 KB total metadata per vector, which also bounds a single non-filterable field; - 2 KB filterable metadata per vector; - 50 metadata keys per vector; - 1024-character vector keys; - at most 10 nonFilterableMetadataKeys of 1 to 63 characters at CreateIndex; - 100 filter constraints per QueryVectors request. A violation is a ValidationException before the engine is called. * s3vectors: accept metadata whose JSON is exactly at the byte limits metadataEntrySize counts a trailing comma for every member, so starting from both braces overcounted by one and rejected metadata that encodes to exactly 2 KB filterable or 100 KB total. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3vectors: rewrap the metadata limits header comment Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
135 lines
4.5 KiB
Go
135 lines
4.5 KiB
Go
package s3vectors
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// AWS S3 Vectors limits on a vector's key and metadata, on an index's
|
|
// metadata configuration, and on a query's filter, from
|
|
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-vectors-limitations.html.
|
|
// Where AWS differs by index mode the ENHANCED value applies: this
|
|
// implementation prefilters, as ENHANCED indexes do, and takes no indexMode.
|
|
const (
|
|
// vectorKeyMaxLength is the PutInputVector key length limit.
|
|
vectorKeyMaxLength = 1024
|
|
// vectorMetadataMaxBytes is the total metadata size per vector, which
|
|
// also bounds a single non-filterable field (100 KB on ENHANCED indexes,
|
|
// 40 KB on CLASSIC).
|
|
vectorMetadataMaxBytes = 100 * 1024
|
|
// vectorFilterableMetadataMaxBytes is the filterable metadata size per vector.
|
|
vectorFilterableMetadataMaxBytes = 2 * 1024
|
|
// vectorMetadataMaxKeys is the number of metadata keys per vector.
|
|
vectorMetadataMaxKeys = 50
|
|
// indexNonFilterableMaxKeys and nonFilterableKeyMaxLength bound
|
|
// MetadataConfiguration.NonFilterableMetadataKeys at CreateIndex.
|
|
indexNonFilterableMaxKeys = 10
|
|
nonFilterableKeyMaxLength = 63
|
|
// queryFilterMaxConstraints is the number of filter constraints per
|
|
// QueryVectors request on an ENHANCED index.
|
|
queryFilterMaxConstraints = 100
|
|
)
|
|
|
|
// filterConstraints counts a filter's constraints: one per comparison on a
|
|
// field, so {"a": 1} is one and {"a": {"$gte": 1, "$lt": 5}} is two;
|
|
// $and, $or and their nesting add none of their own.
|
|
func filterConstraints(filter map[string]interface{}) int {
|
|
n := 0
|
|
for key, cond := range filter {
|
|
if strings.HasPrefix(key, "$") {
|
|
if sub, ok := cond.([]interface{}); ok {
|
|
for _, item := range sub {
|
|
if m, ok := item.(map[string]interface{}); ok {
|
|
n += filterConstraints(m)
|
|
}
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
if ops, ok := cond.(map[string]interface{}); ok && len(ops) > 0 {
|
|
n += len(ops)
|
|
} else {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// checkFilterConstraints applies queryFilterMaxConstraints.
|
|
func checkFilterConstraints(filter map[string]interface{}) error {
|
|
if n := filterConstraints(filter); n > queryFilterMaxConstraints {
|
|
return fmt.Errorf("filter has %d constraints, the maximum is %d", n, queryFilterMaxConstraints)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// validateMetadataConfiguration applies the CreateIndex limits on the
|
|
// non-filterable metadata keys.
|
|
func validateMetadataConfiguration(cfg *MetadataConfiguration) error {
|
|
if cfg == nil {
|
|
return nil
|
|
}
|
|
keys := cfg.NonFilterableMetadataKeys
|
|
if len(keys) > indexNonFilterableMaxKeys {
|
|
return fmt.Errorf("nonFilterableMetadataKeys has %d keys, the maximum is %d", len(keys), indexNonFilterableMaxKeys)
|
|
}
|
|
for _, k := range keys {
|
|
if n := utf8.RuneCountInString(k); n < 1 || n > nonFilterableKeyMaxLength {
|
|
return fmt.Errorf("nonFilterableMetadataKeys entries must be 1 to %d characters", nonFilterableKeyMaxLength)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// checkVectorMetadata applies the per-vector metadata limits. Sizes are the
|
|
// JSON encoding of the metadata, the form the request carried it in; the
|
|
// filterable size counts only the keys the index does not declare
|
|
// non-filterable.
|
|
func checkVectorMetadata(key string, metadata map[string]interface{}, cfg *MetadataConfiguration) error {
|
|
if len(metadata) == 0 {
|
|
return nil
|
|
}
|
|
if len(metadata) > vectorMetadataMaxKeys {
|
|
return fmt.Errorf("vector %q has %d metadata keys, the maximum is %d", key, len(metadata), vectorMetadataMaxKeys)
|
|
}
|
|
var nonFilterable []string
|
|
if cfg != nil {
|
|
nonFilterable = cfg.NonFilterableMetadataKeys
|
|
}
|
|
total, filterable := 1, 1 // the braces, less the last member's comma
|
|
for k, v := range metadata {
|
|
size, err := metadataEntrySize(k, v)
|
|
if err != nil {
|
|
return fmt.Errorf("vector %q metadata %q: %v", key, k, err)
|
|
}
|
|
total += size
|
|
if !slices.Contains(nonFilterable, k) {
|
|
filterable += size
|
|
}
|
|
}
|
|
if total > vectorMetadataMaxBytes {
|
|
return fmt.Errorf("vector %q has %d bytes of metadata, the maximum is %d", key, total, vectorMetadataMaxBytes)
|
|
}
|
|
if filterable > vectorFilterableMetadataMaxBytes {
|
|
return fmt.Errorf("vector %q has %d bytes of filterable metadata, the maximum is %d; declare large keys in nonFilterableMetadataKeys",
|
|
key, filterable, vectorFilterableMetadataMaxBytes)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// metadataEntrySize is the encoded size of one `"key":value,` member.
|
|
func metadataEntrySize(k string, v interface{}) (int, error) {
|
|
encodedKey, err := json.Marshal(k)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
encodedValue, err := json.Marshal(v)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return len(encodedKey) + 1 + len(encodedValue) + 1, nil
|
|
}
|