mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-10 13:58:24 +01:00
Chris Lu
8236c8d136
* vector: expose request metrics and registration-aware health endpoints * vector: report immutable read cache and storage fetch metrics * docs: describe vector engine health, metrics, and maintenance * vector: count canceled requests when their handler is dropped * vector: account for uncached large immutable storage reads * vector: reliably release the stalled credential reader in tests * vector: observe large reads as their streamed body is consumed * vector: bound heartbeats on established master registration streams * test: exercise hardened vector maintenance on Go and Rust volumes * vector: register engines in their configured filer group * vector: make retried index creation preserve compatible datasets * vector: allow registration to advertise a separate address * test: release stalled credential readers without blocking FIFO cleanup * test: allow hardened vector coverage in a named filer group * vector: discover engines in the same filer group as their gateway * vector: fall back when the advertised address is empty * admin: expose vector readiness, maintenance, and management tools (#12520) * s3vectors: share vector discovery with trusted management clients * admin: show vector engine readiness and recent maintenance activity * admin: suggest S3 Vectors IAM actions from the supported operations * admin: manage vector indexes, policies, tags, encryption, and query data * admin: validate SSE-S3 support before creating encrypted vector buckets * admin: discover vector engines in their registration group * admin: probe vector engines concurrently with independent time budgets * admin: verify encryption capability on the remote S3 gateway * admin: defer discovery until after the HTTP server can start * admin: distinguish unknown vector health services from failed engines * admin: count workers that can execute vector index maintenance * admin: allow read-only users to query vectors and inspect tags * vector: scope gateway and admin discovery to the configured filer group * admin: refresh policy state after mutations and serialize policy controls * vector: drain active RPCs before closing replaced discovery connections * vector: respect request cancellation while waiting for discovery * vector: prefer another registered engine after a transport failure * admin: check all masters before reporting no registered vector engines * iam: honor attached vector policies without broadening legacy grants * vector: allow short-lived discovery clients to close their connections * vector: refresh cached master addresses for each discovery lookup * iam: enforce vector bucket tag conditions in attached policies * iam: authorize vector bucket creation with the requested tags * test: exercise vector IAM grants and tag denies on Go and Rust volumes * admin: reject authenticated vector mutations without an admin role * test: run vector IAM assertions before the shared engine is stopped * vector: add native mini startup, shell tools, and a runnable Compose example (#12522) * s3vectors: share vector discovery with trusted management clients * admin: show vector engine readiness and recent maintenance activity * admin: suggest S3 Vectors IAM actions from the supported operations * admin: manage vector indexes, policies, tags, encryption, and query data * admin: validate SSE-S3 support before creating encrypted vector buckets * admin: discover vector engines in their registration group * admin: probe vector engines concurrently with independent time budgets * admin: verify encryption capability on the remote S3 gateway * admin: defer discovery until after the HTTP server can start * admin: distinguish unknown vector health services from failed engines * admin: count workers that can execute vector index maintenance * admin: allow read-only users to query vectors and inspect tags * vector: scope gateway and admin discovery to the configured filer group * admin: refresh policy state after mutations and serialize policy controls * vector: drain active RPCs before closing replaced discovery connections * vector: respect request cancellation while waiting for discovery * vector: prefer another registered engine after a transport failure * admin: check all masters before reporting no registered vector engines * iam: honor attached vector policies without broadening legacy grants * vector: allow short-lived discovery clients to close their connections * vector: refresh cached master addresses for each discovery lookup * iam: enforce vector bucket tag conditions in attached policies * iam: authorize vector bucket creation with the requested tags * test: exercise vector IAM grants and tag denies on Go and Rust volumes * admin: reject authenticated vector mutations without an admin role * test: run vector IAM assertions before the shared engine is stopped * mini: optionally start and supervise the vector engine * mini: bootstrap vector buckets with existing Lance catalog helpers * shell: manage vector buckets, indexes, and vector data * docs: provide a runnable vector and maintenance worker Compose stack * shell: scope vector discovery to the session and close its connections * mini: honor the bind address for vector and metrics listeners * mini: reserve vector ports before assigning gRPC and compatibility listeners * mini: wait for vector storage setup and master registration * mini: create requested vector buckets without requiring the Lance HTTP listener * shell: refresh master addresses during vector discovery * mini: resolve sibling vector executables with platform extensions * vector: preserve uncertain creates and fix mini listener allocation (#12525) * vector: retain catalog records when index creation has an unknown outcome * mini: keep chosen listener ports reserved throughout allocation * mini: resolve hostname binds before starting vector listeners * vector: fix integration test imports and recover interrupted index creation (#12533) * shell: register vector operations from the command package * s3vectors: resume interrupted index initialization safely * mini: exclude disabled WebDAV from listener allocation * s3vectors: retain uncertain creates across refused retries * s3vectors: claim pending indexes before retrying initialization * s3vectors: compare resolved distance metrics during recovery * s3vectors: preserve metadata keys when retrying initialization * s3vectors: verify ownership before initializing new catalog records
102 lines
3.3 KiB
Go
102 lines
3.3 KiB
Go
package command
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"slices"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/lance"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
|
"github.com/seaweedfs/seaweedfs/weed/shell"
|
|
)
|
|
|
|
func init() { shell.Commands = append(shell.Commands, &commandS3Vectors{}) }
|
|
|
|
type commandS3Vectors struct{}
|
|
|
|
func (c *commandS3Vectors) Name() string { return "s3vectors" }
|
|
func (c *commandS3Vectors) HasTag(shell.CommandTag) bool { return false }
|
|
func (c *commandS3Vectors) Help() string {
|
|
return `manage vector buckets, indexes, policies, tags and vectors
|
|
|
|
s3vectors -operation CreateVectorBucket -request '{"vectorBucketName":"embeddings"}'
|
|
s3vectors -operation ListVectorBuckets
|
|
s3vectors -operation CreateIndex -file index.json
|
|
s3vectors -operation QueryVectors -file query.json
|
|
|
|
Requests use the S3 Vectors API JSON shape. The shell uses trusted cluster
|
|
credentials. Index and vector operations require a registered vector engine.
|
|
`
|
|
}
|
|
|
|
func (c *commandS3Vectors) Do(args []string, env *shell.CommandEnv, writer io.Writer) error {
|
|
flags := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
|
|
operation := flags.String("operation", "", "S3 Vectors operation")
|
|
request := flags.String("request", "", "request JSON")
|
|
file := flags.String("file", "", "request JSON file")
|
|
if err := flags.Parse(args); err != nil {
|
|
return err
|
|
}
|
|
if !slices.Contains(s3vectors.Operations(), *operation) {
|
|
return fmt.Errorf("unknown operation %q; supported: %v", *operation, s3vectors.Operations())
|
|
}
|
|
if *request != "" && *file != "" {
|
|
return fmt.Errorf("use either -request or -file")
|
|
}
|
|
body := []byte(*request)
|
|
if *file != "" {
|
|
reader, err := os.Open(*file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer reader.Close()
|
|
body, err = io.ReadAll(io.LimitReader(reader, s3_constants.S3VectorsMaxRequestBodyBytes+1))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if len(body) == 0 {
|
|
body = []byte(`{}`)
|
|
}
|
|
if len(body) > s3_constants.S3VectorsMaxRequestBodyBytes {
|
|
return fmt.Errorf("request is too large")
|
|
}
|
|
if !json.Valid(body) {
|
|
return fmt.Errorf("request must be valid JSON")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
backend := s3vectors.NewVectorDiscovery(env.MasterClient.ListCachedMasters, env.GrpcDialOption(), env.MasterClient.FilerGroup)
|
|
defer backend.Close()
|
|
server := s3vectors.NewServer(env, nil, backend)
|
|
server.SetRegion(s3tables.DefaultRegion)
|
|
server.SetAccountID(s3tables.DefaultAccountID)
|
|
catalog := lance.NewServer(env, nil)
|
|
server.SetDescribe(func(ctx context.Context, id []string) (string, map[string]string, error) {
|
|
r := (&http.Request{Method: http.MethodPost, Header: make(http.Header)}).WithContext(ctx)
|
|
return catalog.DescribeTableInternal(r, id)
|
|
})
|
|
var response json.RawMessage
|
|
if err := server.Execute(ctx, *operation, json.RawMessage(body), &response, s3_constants.AccountAdminId); err != nil {
|
|
return err
|
|
}
|
|
if len(response) > 0 {
|
|
var value interface{}
|
|
if err := json.Unmarshal(response, &value); err != nil {
|
|
return err
|
|
}
|
|
encoder := json.NewEncoder(writer)
|
|
encoder.SetIndent("", " ")
|
|
return encoder.Encode(value)
|
|
}
|
|
return nil
|
|
}
|