mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-27 15:30:45 +01:00
* vector: S3 Vectors API gateway over VectorService weed/s3api/s3vectors serves the AWS S3 Vectors JSON API: bucket CRUD maps to S3 Tables table buckets, index CRUD to Lance table records plus the VectorService CreateIndex/DropIndex RPCs, and the data plane (Put/Get/List/Delete/QueryVectors) calls the shared service. The AWS filter document translates to a Lance SQL predicate over the metadata JSON column; unsupported operators fail loudly rather than overmatch. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * vector: -port.vectors and -vectors.grpc on weed s3 Mirror the Lance server startup: the S3 Vectors API binds its own port and dials the weed-vector-server gRPC address; both disabled by default. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * vector: resolve index ARNs, authorize index ops, fix catalog fields - vector buckets declare FormatLance so a new bucket accepts index tables instead of defaulting to ICEBERG. - ListIndexes decodes the `tables` response field; `tableSummaries` hid every index. - resolveIndexARNs honors indexArn alone; handlers no longer treat indexName as the only carrier. - data-plane RPCs fetch the index record first, so callers authorize against the index resource and a missing index 404s before touching the engine; QueryVectors reports the index's configured metric. - IndexArn added to the index-level request types. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * vector: serve the vectors port over TLS when configured When -key.file/-cert.file are set, the vectors listener mirrors the main API and serves TLS rather than a second plaintext endpoint. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * vector: harden the s3vectors gateway handlers - $ne no longer shadows the operator: exclusion filters emit <> instead of equality. - a null element in a PutVectors list fails the request instead of panicking. - requests are capped at 32 MiB; distances return via *float32 so an exact match still reports 0 when returnDistance is set. - CreateIndex declares the `vector` namespace first and validates dimension (1..=4096) before the catalog write; a failed VectorService.CreateIndex rolls the table record back. - non-filterable metadata keys persist as JSON rather than a comma-joined string, and index records carry their createdAt. - ListVectors honors segmentIndex/segmentCount by slicing the full listing in the gateway. - catalog errors map to AccessDeniedException/ConflictException instead of a blanket ValidationException, and malformed ARNs 400 instead of slicing. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * vector: /{Operation} routes and vectors-port TLS, conn cleanup Path-dispatched clients (POST /QueryVectors) reach the same handlers as X-Amz-Target calls. A failed listener bind now closes the vector gRPC connection instead of leaving it open with the endpoint disabled. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
95 lines
3.1 KiB
Go
95 lines
3.1 KiB
Go
// Package s3vectors serves the AWS S3 Vectors API on top of the shared
|
|
// VectorService gRPC boundary and the S3 Tables catalog.
|
|
package s3vectors
|
|
|
|
import (
|
|
"net/http"
|
|
"strings"
|
|
|
|
"github.com/gorilla/mux"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/vector_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
|
)
|
|
|
|
// FilerClient provides access to the filer for storage operations.
|
|
type FilerClient interface {
|
|
WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error
|
|
}
|
|
|
|
// S3Authenticator authenticates a request, returning the caller identity.
|
|
type S3Authenticator interface {
|
|
AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode)
|
|
DefaultAllow() bool
|
|
}
|
|
|
|
// Server implements the AWS S3 Vectors JSON API.
|
|
type Server struct {
|
|
filerClient FilerClient
|
|
tables *s3tables.Manager
|
|
vectors vector_pb.VectorServiceClient
|
|
authenticator S3Authenticator
|
|
region string
|
|
accountID string
|
|
}
|
|
|
|
// NewServer creates an S3 Vectors server over the given filer and catalog.
|
|
func NewServer(filerClient FilerClient, authenticator S3Authenticator, vectors vector_pb.VectorServiceClient) *Server {
|
|
manager := s3tables.NewManager()
|
|
if authenticator != nil {
|
|
manager.SetDefaultAllow(authenticator.DefaultAllow())
|
|
}
|
|
return &Server{
|
|
filerClient: filerClient,
|
|
tables: manager,
|
|
vectors: vectors,
|
|
authenticator: authenticator,
|
|
}
|
|
}
|
|
|
|
// SetRegion configures the region embedded in the ARNs this server mints.
|
|
func (s *Server) SetRegion(region string) { s.region = region }
|
|
|
|
// SetAccountID configures the account id embedded in the ARNs this server mints.
|
|
func (s *Server) SetAccountID(id string) { s.accountID = id }
|
|
|
|
// RegisterRoutes mounts POST / dispatching on the X-Amz-Target header and
|
|
// POST /{Operation} for clients that route per-path.
|
|
func (s *Server) RegisterRoutes(router *mux.Router) {
|
|
router.HandleFunc("/", s.Auth(s.dispatch)).Methods(http.MethodPost)
|
|
router.HandleFunc("/{op}", s.Auth(s.dispatch)).Methods(http.MethodPost)
|
|
}
|
|
|
|
func (s *Server) dispatch(w http.ResponseWriter, r *http.Request) {
|
|
// Clients carry the operation in the X-Amz-Target header or in the path —
|
|
// POST /QueryVectors and POST / with a target mean the same call here.
|
|
op := strings.Trim(r.URL.Path, "/")
|
|
if i := strings.LastIndex(op, "/"); i >= 0 {
|
|
op = op[i+1:]
|
|
}
|
|
if op == "" {
|
|
target := r.Header.Get("X-Amz-Target")
|
|
op = target[strings.LastIndex(target, ".")+1:]
|
|
}
|
|
handler, ok := handlers[op]
|
|
if !ok {
|
|
writeError(w, http.StatusBadRequest, "ValidationException", "unknown operation "+op)
|
|
return
|
|
}
|
|
handler(s, w, r)
|
|
}
|
|
|
|
// Auth authenticates the caller and puts the identity in the request context.
|
|
func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
identity, _, errCode := s.authenticator.AuthenticateRequest(r)
|
|
if errCode != s3err.ErrNone {
|
|
writeError(w, http.StatusForbidden, "AccessDenied", "authentication failed")
|
|
return
|
|
}
|
|
r = r.WithContext(contextWithIdentity(r.Context(), identity))
|
|
handler(w, r)
|
|
}
|
|
}
|