vector: VectorService gRPC contract (P1) (#12101)

* vector: VectorService protobuf contract

The shared RPC boundary between the Go S3 gateway and the Rust vector
engine from the vector design doc: CreateIndex/DropIndex, Put/Get/
List/Delete/Query vectors. A Table reference carries the Lance table id
and an optional pinned version, matching the snapshot discipline queries
are required to keep.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* vector: generated Go stubs for vector.proto

protoc output via weed/pb/Makefile.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* vector: carry return_data through QueryVectors

S3 Vectors QueryVectors honours returnData, but the contract had no
way to ask for or carry vector data in a query response, so the
gateway would need a second GetVectors round-trip per query. Add
return_data to the request and data to QueryResult now, while the
contract is still new.

* vector: pin metadata numbers to double precision in the contract

Metadata arrives as JSON, where numbers carry double precision; say
so on the field so neither side promises integer fidelity past 2^53.

* vector: clarify the query filter is raw Lance SQL over one metadata column

The doc said 'metadata columns' but the table stores all metadata in a
single JSON column named metadata; the gateway translates S3 Vectors
filter expressions onto it.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

weed/pb/Makefile

@@ -17,6 +17,7 @@ gen:
 	protoc mq_broker.proto --go_out=./mq_pb --go-grpc_out=./mq_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
 	protoc mq_schema.proto --go_out=./schema_pb --go-grpc_out=./schema_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
 	protoc mq_agent.proto --go_out=./mq_agent_pb --go-grpc_out=./mq_agent_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	protoc vector.proto --go_out=./vector_pb --go-grpc_out=./vector_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
 	protoc worker.proto --go_out=./worker_pb --go-grpc_out=./worker_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
 	mkdir -p ./plugin_pb
 	protoc plugin.proto --go_out=./plugin_pb --go-grpc_out=./plugin_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative

weed/pb/vector.proto

@@ -0,0 +1,127 @@
+syntax = "proto3";
+
+package vector_pb;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/vector_pb";
+
+import "google/protobuf/struct.proto";
+
+// The shared contract between the Go S3 gateway and the vector query engine.
+// The public S3 Vectors API stays an AWS contract; this RPC is the internal
+// boundary the design doc gives both editions.
+service VectorService {
+  rpc CreateIndex(CreateIndexRequest) returns (CreateIndexResponse);
+  rpc DropIndex(DropIndexRequest) returns (DropIndexResponse);
+  rpc PutVectors(PutVectorsRequest) returns (PutVectorsResponse);
+  rpc GetVectors(GetVectorsRequest) returns (GetVectorsResponse);
+  rpc DeleteVectors(DeleteVectorsRequest) returns (DeleteVectorsResponse);
+  rpc ListVectors(ListVectorsRequest) returns (ListVectorsResponse);
+  rpc QueryVectors(QueryVectorsRequest) returns (QueryVectorsResponse);
+}
+
+enum DistanceMetric {
+  DISTANCE_METRIC_UNSPECIFIED = 0;
+  L2 = 1;
+  COSINE = 2;
+  DOT_PRODUCT = 3;
+}
+
+// Table addresses a Lance table through the namespace: the id parts as the
+// namespace catalog joins them.
+message Table {
+  repeated string id = 1;
+  // Dataset version to pin reads to; 0 means latest.
+  uint64 version = 2;
+}
+
+message CreateIndexRequest {
+  Table table = 1;
+  uint32 dimension = 2;
+  DistanceMetric distance_metric = 3;
+  repeated string non_filterable_metadata_keys = 4;
+}
+
+message CreateIndexResponse {
+  uint64 version = 1;
+}
+
+message DropIndexRequest {
+  Table table = 1;
+}
+
+message DropIndexResponse {}
+
+message Vector {
+  string key = 1;
+  repeated float data = 2;
+  // JSON-typed metadata: numbers carry double precision, as on the wire.
+  google.protobuf.Struct metadata = 3;
+}
+
+message PutVectorsRequest {
+  Table table = 1;
+  repeated Vector vectors = 2;
+}
+
+message PutVectorsResponse {
+  uint64 version = 1;
+}
+
+message GetVectorsRequest {
+  Table table = 1;
+  repeated string keys = 2;
+  bool return_data = 3;
+  bool return_metadata = 4;
+}
+
+message GetVectorsResponse {
+  repeated Vector vectors = 1;
+}
+
+message DeleteVectorsRequest {
+  Table table = 1;
+  repeated string keys = 2;
+}
+
+message DeleteVectorsResponse {
+  uint64 version = 1;
+}
+
+message ListVectorsRequest {
+  Table table = 1;
+  uint32 max_results = 2;
+  string next_token = 3;
+  bool return_data = 4;
+  bool return_metadata = 5;
+}
+
+message ListVectorsResponse {
+  repeated Vector vectors = 1;
+  string next_token = 2;
+}
+
+message QueryVectorsRequest {
+  Table table = 1;
+  repeated float query_vector = 2;
+  uint32 top_k = 3;
+  // Lance SQL filter; empty means unfiltered. Metadata is one JSON
+  // `metadata` column, so the gateway translates S3 Vectors filter
+  // expressions onto it before they land here.
+  string filter = 4;
+  // Lance's post-filter escape hatch; off prefilters, matching S3 Vectors.
+  bool postfilter = 5;
+  bool return_metadata = 6;
+  bool return_distance = 7;
+  bool return_data = 8;
+}
+
+message QueryResult {
+  string key = 1;
+  float distance = 2;
+  google.protobuf.Struct metadata = 3;
+  repeated float data = 4;
+}
+
+message QueryVectorsResponse {
+  repeated QueryResult vectors = 1;
+}

weed/pb/vector_pb/vector_grpc.pb.go

@@ -0,0 +1,357 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.6.2
+// - protoc             v3.21.12
+// source: vector.proto
+
+package vector_pb
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.64.0 or later.
+const _ = grpc.SupportPackageIsVersion9
+
+const (
+	VectorService_CreateIndex_FullMethodName   = "/vector_pb.VectorService/CreateIndex"
+	VectorService_DropIndex_FullMethodName     = "/vector_pb.VectorService/DropIndex"
+	VectorService_PutVectors_FullMethodName    = "/vector_pb.VectorService/PutVectors"
+	VectorService_GetVectors_FullMethodName    = "/vector_pb.VectorService/GetVectors"
+	VectorService_DeleteVectors_FullMethodName = "/vector_pb.VectorService/DeleteVectors"
+	VectorService_ListVectors_FullMethodName   = "/vector_pb.VectorService/ListVectors"
+	VectorService_QueryVectors_FullMethodName  = "/vector_pb.VectorService/QueryVectors"
+)
+
+// VectorServiceClient is the client API for VectorService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+//
+// The shared contract between the Go S3 gateway and the vector query engine.
+// The public S3 Vectors API stays an AWS contract; this RPC is the internal
+// boundary the design doc gives both editions.
+type VectorServiceClient interface {
+	CreateIndex(ctx context.Context, in *CreateIndexRequest, opts ...grpc.CallOption) (*CreateIndexResponse, error)
+	DropIndex(ctx context.Context, in *DropIndexRequest, opts ...grpc.CallOption) (*DropIndexResponse, error)
+	PutVectors(ctx context.Context, in *PutVectorsRequest, opts ...grpc.CallOption) (*PutVectorsResponse, error)
+	GetVectors(ctx context.Context, in *GetVectorsRequest, opts ...grpc.CallOption) (*GetVectorsResponse, error)
+	DeleteVectors(ctx context.Context, in *DeleteVectorsRequest, opts ...grpc.CallOption) (*DeleteVectorsResponse, error)
+	ListVectors(ctx context.Context, in *ListVectorsRequest, opts ...grpc.CallOption) (*ListVectorsResponse, error)
+	QueryVectors(ctx context.Context, in *QueryVectorsRequest, opts ...grpc.CallOption) (*QueryVectorsResponse, error)
+}
+
+type vectorServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewVectorServiceClient(cc grpc.ClientConnInterface) VectorServiceClient {
+	return &vectorServiceClient{cc}
+}
+
+func (c *vectorServiceClient) CreateIndex(ctx context.Context, in *CreateIndexRequest, opts ...grpc.CallOption) (*CreateIndexResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(CreateIndexResponse)
+	err := c.cc.Invoke(ctx, VectorService_CreateIndex_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *vectorServiceClient) DropIndex(ctx context.Context, in *DropIndexRequest, opts ...grpc.CallOption) (*DropIndexResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(DropIndexResponse)
+	err := c.cc.Invoke(ctx, VectorService_DropIndex_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *vectorServiceClient) PutVectors(ctx context.Context, in *PutVectorsRequest, opts ...grpc.CallOption) (*PutVectorsResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(PutVectorsResponse)
+	err := c.cc.Invoke(ctx, VectorService_PutVectors_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *vectorServiceClient) GetVectors(ctx context.Context, in *GetVectorsRequest, opts ...grpc.CallOption) (*GetVectorsResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetVectorsResponse)
+	err := c.cc.Invoke(ctx, VectorService_GetVectors_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *vectorServiceClient) DeleteVectors(ctx context.Context, in *DeleteVectorsRequest, opts ...grpc.CallOption) (*DeleteVectorsResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(DeleteVectorsResponse)
+	err := c.cc.Invoke(ctx, VectorService_DeleteVectors_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *vectorServiceClient) ListVectors(ctx context.Context, in *ListVectorsRequest, opts ...grpc.CallOption) (*ListVectorsResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ListVectorsResponse)
+	err := c.cc.Invoke(ctx, VectorService_ListVectors_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *vectorServiceClient) QueryVectors(ctx context.Context, in *QueryVectorsRequest, opts ...grpc.CallOption) (*QueryVectorsResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(QueryVectorsResponse)
+	err := c.cc.Invoke(ctx, VectorService_QueryVectors_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// VectorServiceServer is the server API for VectorService service.
+// All implementations must embed UnimplementedVectorServiceServer
+// for forward compatibility.
+//
+// The shared contract between the Go S3 gateway and the vector query engine.
+// The public S3 Vectors API stays an AWS contract; this RPC is the internal
+// boundary the design doc gives both editions.
+type VectorServiceServer interface {
+	CreateIndex(context.Context, *CreateIndexRequest) (*CreateIndexResponse, error)
+	DropIndex(context.Context, *DropIndexRequest) (*DropIndexResponse, error)
+	PutVectors(context.Context, *PutVectorsRequest) (*PutVectorsResponse, error)
+	GetVectors(context.Context, *GetVectorsRequest) (*GetVectorsResponse, error)
+	DeleteVectors(context.Context, *DeleteVectorsRequest) (*DeleteVectorsResponse, error)
+	ListVectors(context.Context, *ListVectorsRequest) (*ListVectorsResponse, error)
+	QueryVectors(context.Context, *QueryVectorsRequest) (*QueryVectorsResponse, error)
+	mustEmbedUnimplementedVectorServiceServer()
+}
+
+// UnimplementedVectorServiceServer must be embedded to have
+// forward compatible implementations.
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedVectorServiceServer struct{}
+
+func (UnimplementedVectorServiceServer) CreateIndex(context.Context, *CreateIndexRequest) (*CreateIndexResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "method CreateIndex not implemented")
+}
+func (UnimplementedVectorServiceServer) DropIndex(context.Context, *DropIndexRequest) (*DropIndexResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "method DropIndex not implemented")
+}
+func (UnimplementedVectorServiceServer) PutVectors(context.Context, *PutVectorsRequest) (*PutVectorsResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "method PutVectors not implemented")
+}
+func (UnimplementedVectorServiceServer) GetVectors(context.Context, *GetVectorsRequest) (*GetVectorsResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "method GetVectors not implemented")
+}
+func (UnimplementedVectorServiceServer) DeleteVectors(context.Context, *DeleteVectorsRequest) (*DeleteVectorsResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "method DeleteVectors not implemented")
+}
+func (UnimplementedVectorServiceServer) ListVectors(context.Context, *ListVectorsRequest) (*ListVectorsResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "method ListVectors not implemented")
+}
+func (UnimplementedVectorServiceServer) QueryVectors(context.Context, *QueryVectorsRequest) (*QueryVectorsResponse, error) {
+	return nil, status.Error(codes.Unimplemented, "method QueryVectors not implemented")
+}
+func (UnimplementedVectorServiceServer) mustEmbedUnimplementedVectorServiceServer() {}
+func (UnimplementedVectorServiceServer) testEmbeddedByValue()                       {}
+
+// UnsafeVectorServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to VectorServiceServer will
+// result in compilation errors.
+type UnsafeVectorServiceServer interface {
+	mustEmbedUnimplementedVectorServiceServer()
+}
+
+func RegisterVectorServiceServer(s grpc.ServiceRegistrar, srv VectorServiceServer) {
+	// If the following call panics, it indicates UnimplementedVectorServiceServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
+	s.RegisterService(&VectorService_ServiceDesc, srv)
+}
+
+func _VectorService_CreateIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CreateIndexRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(VectorServiceServer).CreateIndex(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: VectorService_CreateIndex_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(VectorServiceServer).CreateIndex(ctx, req.(*CreateIndexRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _VectorService_DropIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(DropIndexRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(VectorServiceServer).DropIndex(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: VectorService_DropIndex_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(VectorServiceServer).DropIndex(ctx, req.(*DropIndexRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _VectorService_PutVectors_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(PutVectorsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(VectorServiceServer).PutVectors(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: VectorService_PutVectors_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(VectorServiceServer).PutVectors(ctx, req.(*PutVectorsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _VectorService_GetVectors_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetVectorsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(VectorServiceServer).GetVectors(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: VectorService_GetVectors_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(VectorServiceServer).GetVectors(ctx, req.(*GetVectorsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _VectorService_DeleteVectors_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(DeleteVectorsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(VectorServiceServer).DeleteVectors(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: VectorService_DeleteVectors_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(VectorServiceServer).DeleteVectors(ctx, req.(*DeleteVectorsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _VectorService_ListVectors_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ListVectorsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(VectorServiceServer).ListVectors(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: VectorService_ListVectors_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(VectorServiceServer).ListVectors(ctx, req.(*ListVectorsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _VectorService_QueryVectors_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(QueryVectorsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(VectorServiceServer).QueryVectors(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: VectorService_QueryVectors_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(VectorServiceServer).QueryVectors(ctx, req.(*QueryVectorsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// VectorService_ServiceDesc is the grpc.ServiceDesc for VectorService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var VectorService_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "vector_pb.VectorService",
+	HandlerType: (*VectorServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "CreateIndex",
+			Handler:    _VectorService_CreateIndex_Handler,
+		},
+		{
+			MethodName: "DropIndex",
+			Handler:    _VectorService_DropIndex_Handler,
+		},
+		{
+			MethodName: "PutVectors",
+			Handler:    _VectorService_PutVectors_Handler,
+		},
+		{
+			MethodName: "GetVectors",
+			Handler:    _VectorService_GetVectors_Handler,
+		},
+		{
+			MethodName: "DeleteVectors",
+			Handler:    _VectorService_DeleteVectors_Handler,
+		},
+		{
+			MethodName: "ListVectors",
+			Handler:    _VectorService_ListVectors_Handler,
+		},
+		{
+			MethodName: "QueryVectors",
+			Handler:    _VectorService_QueryVectors_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "vector.proto",
+}