diff --git a/weed/pb/vector.proto b/weed/pb/vector.proto index 723aefed90..b145969a23 100644 --- a/weed/pb/vector.proto +++ b/weed/pb/vector.proto @@ -108,6 +108,11 @@ message QueryVectorsRequest { // `metadata` column, so the gateway translates S3 Vectors filter // expressions onto it before they land here. string filter = 4; + // The raw S3 Vectors filter document. Indexes whose metadata column + // predates lance's jsonb storage cannot plan json_get_* calls; the engine + // evaluates this row-wise instead and constrains the scan by key. Gateways + // still send `filter` for engines that only know the SQL form. + string filter_json = 9; // Lance's post-filter escape hatch; off prefilters, matching S3 Vectors. bool postfilter = 5; bool return_metadata = 6; diff --git a/weed/pb/vector_pb/vector.pb.go b/weed/pb/vector_pb/vector.pb.go index 83b69c70d0..0a46208972 100644 --- a/weed/pb/vector_pb/vector.pb.go +++ b/weed/pb/vector_pb/vector.pb.go @@ -823,6 +823,11 @@ type QueryVectorsRequest struct { // `metadata` column, so the gateway translates S3 Vectors filter // expressions onto it before they land here. Filter string `protobuf:"bytes,4,opt,name=filter,proto3" json:"filter,omitempty"` + // The raw S3 Vectors filter document. Indexes whose metadata column + // predates lance's jsonb storage cannot plan json_get_* calls; the engine + // evaluates this row-wise instead and constrains the scan by key. Gateways + // still send `filter` for engines that only know the SQL form. + FilterJson string `protobuf:"bytes,9,opt,name=filter_json,json=filterJson,proto3" json:"filter_json,omitempty"` // Lance's post-filter escape hatch; off prefilters, matching S3 Vectors. Postfilter bool `protobuf:"varint,5,opt,name=postfilter,proto3" json:"postfilter,omitempty"` ReturnMetadata bool `protobuf:"varint,6,opt,name=return_metadata,json=returnMetadata,proto3" json:"return_metadata,omitempty"` @@ -890,6 +895,13 @@ func (x *QueryVectorsRequest) GetFilter() string { return "" } +func (x *QueryVectorsRequest) GetFilterJson() string { + if x != nil { + return x.FilterJson + } + return "" +} + func (x *QueryVectorsRequest) GetPostfilter() bool { if x != nil { return x.Postfilter @@ -1082,12 +1094,14 @@ const file_vector_proto_rawDesc = "" + "\x13ListVectorsResponse\x12+\n" + "\avectors\x18\x01 \x03(\v2\x11.vector_pb.VectorR\avectors\x12\x1d\n" + "\n" + - "next_token\x18\x02 \x01(\tR\tnextToken\"\xa0\x02\n" + + "next_token\x18\x02 \x01(\tR\tnextToken\"\xc1\x02\n" + "\x13QueryVectorsRequest\x12&\n" + "\x05table\x18\x01 \x01(\v2\x10.vector_pb.TableR\x05table\x12!\n" + "\fquery_vector\x18\x02 \x03(\x02R\vqueryVector\x12\x13\n" + "\x05top_k\x18\x03 \x01(\rR\x04topK\x12\x16\n" + - "\x06filter\x18\x04 \x01(\tR\x06filter\x12\x1e\n" + + "\x06filter\x18\x04 \x01(\tR\x06filter\x12\x1f\n" + + "\vfilter_json\x18\t \x01(\tR\n" + + "filterJson\x12\x1e\n" + "\n" + "postfilter\x18\x05 \x01(\bR\n" + "postfilter\x12'\n" + diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index 583f51cb90..25c5816bbb 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -899,11 +899,16 @@ func (s *Server) handleQueryVectors(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", "topK must be non-negative") return } + var filterJSON []byte + if len(req.Filter) > 0 { + filterJSON, _ = json.Marshal(req.Filter) + } resp, err := s.vectors.QueryVectors(r.Context(), &vector_pb.QueryVectorsRequest{ Table: tableID(arns.bucket, arns.index), QueryVector: queryVector, TopK: uint32(req.TopK), Filter: filter, + FilterJson: string(filterJSON), ReturnMetadata: req.ReturnMetadata, ReturnDistance: req.ReturnDistance, })