diff --git a/.github/workflows/rust-vector-tests.yml b/.github/workflows/rust-vector-tests.yml index a22817ab24..437402e3cc 100644 --- a/.github/workflows/rust-vector-tests.yml +++ b/.github/workflows/rust-vector-tests.yml @@ -87,22 +87,17 @@ jobs: # volume, filer, gateway, engine) under mutual TLS, read JWTs # required, and every chunk encrypted by the gateway. hardened: ["", "1"] - worker: [""] + worker: ["", "1"] # Worker: the admin server and the Lance maintenance worker join the # cluster, for the tests that build vector indexes through a job. - include: - - volume: go - hardened: "" - worker: "1" - - volume: rust - hardened: "" - worker: "1" + env: CARGO_PROFILE_DEV_DEBUG: "0" CARGO_BUILD_JOBS: "2" CARGO_TARGET_DIR: ${{ github.workspace }}/seaweed-vector/target VOLUME_SERVER_IMPL: ${{ matrix.volume }} VECTOR_QUERY_MODE: single + VECTOR_FILER_GROUP: vectors-ci VECTOR_TLS: ${{ matrix.hardened }} VECTOR_READ_JWT: ${{ matrix.hardened }} VECTOR_ENCRYPT_VOLUMES: ${{ matrix.hardened }} diff --git a/docker/s3vectors/README.md b/docker/s3vectors/README.md new file mode 100644 index 0000000000..d0cb14473d --- /dev/null +++ b/docker/s3vectors/README.md @@ -0,0 +1,39 @@ +# Vector buckets locally + +Run on Linux containers (amd64 or arm64): + +```sh +docker compose up -d +export AWS_ACCESS_KEY_ID=vector-demo +export AWS_SECRET_ACCESS_KEY=vector-demo-secret +export AWS_DEFAULT_REGION=us-east-1 +aws --endpoint-url http://localhost:8333 s3vectors create-index \ + --vector-bucket-name embeddings --index-name images \ + --data-type float32 --dimension 3 --distance-metric cosine +aws --endpoint-url http://localhost:8333 s3vectors put-vectors \ + --vector-bucket-name embeddings --index-name images \ + --vectors '[{"key":"first","data":{"float32":[1,0,0]}}]' +aws --endpoint-url http://localhost:8333 s3vectors query-vectors \ + --vector-bucket-name embeddings --index-name images \ + --query-vector '{"float32":[1,0,0]}' --top-k 1 +``` + +The engine registers with the master. Wait for that registration before creating +an index. The Lance worker builds and refreshes ANN indexes as data grows; +queries can scan data before an ANN index is built. The demo's admin UI is at +http://localhost:12535 (user `admin`, password `vector-demo`). Demo credentials +and plaintext cluster connections are for this local example. Production +settings are in the Helm/operator TLS and authentication documentation. + +`docker compose down` preserves the named data volume. Add `-v` to discard it. +Override `SEAWEEDFS_IMAGE` to test a newer image. With metrics-enabled engines, +append `--metrics-port=9329 --metrics-ip=0.0.0.0` to the vector command and scrape +`vector:9329/metrics` inside the Compose network. + +A native development setup can use `weed mini -vector -vectorBucket=embeddings` +with `weed-vector` beside `weed` or on PATH. `-vector.binary` selects an explicit +binary, and `-vector.metricsPort=9329` enables HTTP readiness and metrics on a +compatible engine. The child inherits the cluster's configuration directory and +stops with mini. Run `weed-worker` separately for ANN maintenance. `weed shell` +provides `s3vectors -operation -file request.json` for the same API +operations using trusted cluster credentials. diff --git a/docker/s3vectors/compose.yaml b/docker/s3vectors/compose.yaml new file mode 100644 index 0000000000..926474466c --- /dev/null +++ b/docker/s3vectors/compose.yaml @@ -0,0 +1,34 @@ +services: + seaweedfs: + image: ${SEAWEEDFS_IMAGE:-chrislusf/seaweedfs:4.53} + command: mini -dir=/data -ip=seaweedfs -ip.bind=0.0.0.0 -admin.password=vector-demo -admin.worker.ip=0.0.0.0 -tableBucket=embeddings:LANCE + environment: &environment + WEED_JWT_FILER_SIGNING_KEY: vector-demo-cluster-key + AWS_ACCESS_KEY_ID: vector-demo + AWS_SECRET_ACCESS_KEY: vector-demo-secret + ports: + - "127.0.0.1:8333:8333" + - "127.0.0.1:12535:12535" + volumes: + - data:/data + healthcheck: + test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status"] + interval: 5s + timeout: 3s + retries: 30 + vector: + image: ${SEAWEEDFS_IMAGE:-chrislusf/seaweedfs:4.53} + command: vector --listen=vector:8090 --masters=seaweedfs:9333 + environment: *environment + depends_on: + seaweedfs: + condition: service_healthy + worker: + image: ${SEAWEEDFS_IMAGE:-chrislusf/seaweedfs:4.53} + command: worker-rust --admin=seaweedfs:12535 --masters=seaweedfs:9333 --namespace=http://seaweedfs:9101 + environment: *environment + depends_on: + seaweedfs: + condition: service_healthy +volumes: + data: diff --git a/seaweed-vector/Cargo.lock b/seaweed-vector/Cargo.lock index a293806dc1..c01416032f 100644 --- a/seaweed-vector/Cargo.lock +++ b/seaweed-vector/Cargo.lock @@ -6684,6 +6684,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tonic-health" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcfab99db777fba2802f0dfa861d1628d1ae916fb199d29819941f139ae85082" +dependencies = [ + "prost", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", +] + [[package]] name = "tonic-prost" version = "0.14.6" @@ -7217,6 +7230,7 @@ dependencies = [ "arrow-cast", "arrow-schema", "async-trait", + "axum", "bytes", "clap", "futures", @@ -7225,6 +7239,7 @@ dependencies = [ "lance-linalg", "moka", "object_store", + "prometheus", "prost", "prost-types", "protoc-bin-vendored", @@ -7236,6 +7251,7 @@ dependencies = [ "tokio", "tokio-stream", "tonic", + "tonic-health", "tonic-prost", "tonic-prost-build", "tracing", diff --git a/seaweed-vector/README.md b/seaweed-vector/README.md new file mode 100644 index 0000000000..7c0bec281c --- /dev/null +++ b/seaweed-vector/README.md @@ -0,0 +1,40 @@ +# SeaweedFS vector engine + +`weed-vector` serves the S3 Vectors data plane through the S3 gateway. Indexes use Lance tables on the cluster's filers and volume servers. Go and Rust volume servers use the same engine and API. + +With a local `weed mini` cluster running: + +```sh +weed-vector --masters=localhost:9333 --metrics-port=9329 +``` + +Point an AWS S3 Vectors client at the S3 endpoint, normally `http://localhost:8333`. The engine registers with the masters automatically. `--listen` defaults to `127.0.0.1:8090`; use an address reachable from the gateway when running on another host or in a container. + +| Setting | Default | Purpose | +| --- | --- | --- | +| `--masters` | Required | Comma-separated master addresses | +| `--listen` | `127.0.0.1:8090` | gRPC listener and registered address | +| `--filer-group` | Empty | Filer group used for table storage | +| `--read-cache-mb` | `512` | Immutable data/index read cache; zero disables it | +| `--nprobes` | Automatic | ANN partitions searched per query | +| `--metrics-port` | `0` | HTTP metrics and health listener; zero disables it | +| `--metrics-ip` | `127.0.0.1` | HTTP listener address | +| `--security-toml` | Discovered | Cluster certificates, signing keys and encryption keys | + +The HTTP listener provides `/health`, `/ready`, and `/metrics`. Readiness requires an active registration with the master. The gRPC listener also implements the standard health service for `vector_pb.VectorService` and the whole server. These health statuses follow master registration. + +Metrics use the `SeaweedFS_vector_` prefix: + +- `connected`: active master registration. +- `request_total{type,code}`, `request_seconds{type}`, and `in_flight_requests{type}`: engine requests and their gRPC outcomes. +- `query_results_total`: vectors returned by queries. +- `cache_requests_total{result}`, `cache_bytes`, and `cache_entries`: immutable read cache activity and occupancy. +- `storage_read_bytes_total` and `storage_read_seconds`: storage fetches for cacheable immutable reads. + +The S3 gateway separately publishes `SeaweedFS_s3_` request metrics with `type="s3vectors."`. Scrape both processes to distinguish gateway latency from engine work. Metrics and HTTP probes are unauthenticated; bind them to an internal address or restrict their network access. + +For mutual TLS, configure `[grpc.vector]` for the engine's server certificate, `[grpc.client]` for its master/filer connections, and the cluster CA. HTTPS volume reads also use `[https.client]`. The engine follows the same signing and encryption configuration as the other cluster clients. + +Run the Lance maintenance worker against the same masters and Lance namespace to build and update ANN indexes. Queries can scan data that has not been indexed yet. See the [maintenance worker documentation](../seaweed-worker/README.md) and [Helm chart](../k8s/charts/seaweedfs/README.md) for deployment settings. + +The integration suite at `test/s3vectors` runs the same API and observability checks against both Go and Rust volume servers, including mutual TLS, signed volume reads, and encrypted volume data. diff --git a/seaweed-vector/crates/server/Cargo.toml b/seaweed-vector/crates/server/Cargo.toml index 6fb8743c72..cbcf163291 100644 --- a/seaweed-vector/crates/server/Cargo.toml +++ b/seaweed-vector/crates/server/Cargo.toml @@ -13,6 +13,9 @@ name = "weed-vector" path = "src/main.rs" [dependencies] +axum = "0.8" +prometheus = { version = "0.13", default-features = false } +tonic-health = "0.14" weed-vector-core = { path = "../core" } weed-lance-worker = { path = "../../../seaweed-worker/crates/lance" } sw-client = { path = "../../../seaweed-worker/crates/sw-client" } diff --git a/seaweed-vector/crates/server/src/lib.rs b/seaweed-vector/crates/server/src/lib.rs index 2215c11b1a..99624729d9 100644 --- a/seaweed-vector/crates/server/src/lib.rs +++ b/seaweed-vector/crates/server/src/lib.rs @@ -1,5 +1,7 @@ //! The VectorService gRPC server the S3 gateway talks to. +pub mod metrics; +pub mod observed; pub mod read_cache; pub mod service; diff --git a/seaweed-vector/crates/server/src/main.rs b/seaweed-vector/crates/server/src/main.rs index 1144bfcd84..f181220f20 100644 --- a/seaweed-vector/crates/server/src/main.rs +++ b/seaweed-vector/crates/server/src/main.rs @@ -5,12 +5,16 @@ use clap::Parser; use tokio_stream::wrappers::TcpListenerStream; use tonic::transport::Server; use weed_lance_worker::cluster::Cluster; -use weed_vector_server::master_client; use weed_vector_server::pb::vector::vector_service_server::VectorServiceServer; use weed_vector_server::read_cache::ReadCache; use weed_vector_server::security::Security; use weed_vector_server::service::VectorServer; use weed_vector_server::tls::{self, ServerTls}; +use weed_vector_server::{ + master_client, + metrics::{self, Metrics}, + observed::ObservedService, +}; /// The VectorService the S3 gateway fans vector calls out to. #[derive(Parser)] @@ -18,6 +22,9 @@ struct Args { /// gRPC listen address. #[arg(long, env = "WEED_VECTOR_LISTEN", default_value = "127.0.0.1:8090")] listen: String, + /// Address to register when it differs from the listen address. + #[arg(long, env = "WEED_VECTOR_ADVERTISE")] + advertise: Option, /// Comma-separated master addresses. The engine registers with them, so /// the gateway finds it, and asks them for the filers it keeps tables /// through. @@ -45,6 +52,21 @@ struct Args { /// processes find in -config_dir, ., ~/.seaweedfs and /etc/seaweedfs. #[arg(long, env = "WEED_VECTOR_SECURITY_TOML")] security_toml: Option, + #[arg( + long, + alias = "metricsPort", + env = "WEED_VECTOR_METRICS_PORT", + default_value_t = 0 + )] + metrics_port: u16, + #[arg( + long, + alias = "metricsIp", + env = "WEED_VECTOR_METRICS_IP", + default_value = "127.0.0.1" + )] + metrics_ip: String, + /// The directory weed's -config_dir names, searched for security.toml /// first. #[arg(long, alias = "config_dir", env = "WEED_VECTOR_CONFIG_DIR")] @@ -78,7 +100,13 @@ fn normalize_args(args: Vec) -> Vec { /// A wildcard listen address is not reachable for peers; derive the outbound /// IP toward the first master instead — the address the master sees anyway. -fn advertised_addr(listen: &str, masters: &[String]) -> String { +fn advertised_addr(listen: &str, masters: &[String], advertise: Option<&str>) -> String { + if let Some(address) = advertise + .map(str::trim) + .filter(|address| !address.is_empty()) + { + return address.to_string(); + } let Some((host, _)) = listen.rsplit_once(':') else { return listen.to_string(); }; @@ -128,11 +156,31 @@ async fn main() -> Result<()> { .with_context(|| format!("listen on {}", args.listen))?; let cluster = Cluster::new(&args.masters, &args.filer_group, &security)?; let reads = cluster.storage(&security).await?; + let (health, health_service) = tonic_health::server::health_reporter(); + let metrics = Metrics::new()?.with_health(health); + metrics.set_connected(false).await; + let cache = (args.read_cache_mb > 0) + .then(|| ReadCache::new(args.read_cache_mb << 20).with_metrics(metrics.clone())); let mut service = VectorServer::new() .with_nprobes(args.nprobes.map(std::num::NonZeroUsize::get)) .with_storage(reads); - if args.read_cache_mb > 0 { - service = service.with_read_cache(ReadCache::new(args.read_cache_mb << 20)); + if let Some(cache) = &cache { + service = service.with_read_cache(cache.clone()); + } + if args.metrics_port > 0 { + let ip = args + .metrics_ip + .trim_start_matches('[') + .trim_end_matches(']') + .parse::()?; + let listener = + tokio::net::TcpListener::bind(std::net::SocketAddr::new(ip, args.metrics_port)).await?; + let metrics = metrics.clone(); + tokio::spawn(async move { + if let Err(error) = metrics::serve_listener(metrics, listener, cache).await { + tracing::error!(%error, "vector metrics server stopped"); + } + }); } tracing::info!( listen = %args.listen, @@ -148,19 +196,25 @@ async fn main() -> Result<()> { ); // Registered only now, with storage connected and the listener bound, // so the gateway never finds an engine that cannot serve yet. - let advertise = advertised_addr(&args.listen, &cluster.masters); + let advertise = advertised_addr(&args.listen, &cluster.masters, args.advertise.as_deref()); tokio::spawn(master_client::keep_connected( cluster.masters.clone(), advertise, + args.filer_group.clone(), cluster.master_tls.clone(), + metrics.clone(), )); let router = Server::builder() + .add_service(health_service) // Match the gateway's 1 GiB cap — put batches over the tonic 4 MiB // default otherwise get rejected before they reach the handler. .add_service( - VectorServiceServer::new(service) - .max_decoding_message_size(1 << 30) - .max_encoding_message_size(1 << 30), + VectorServiceServer::new(ObservedService { + inner: service, + metrics, + }) + .max_decoding_message_size(1 << 30) + .max_encoding_message_size(1 << 30), ); match server_tls { Some(tls) => { @@ -180,6 +234,20 @@ async fn main() -> Result<()> { mod tests { use super::*; + #[test] + fn an_empty_advertise_uses_the_listen_address() { + for value in [None, Some(""), Some(" ")] { + assert_eq!( + advertised_addr("127.0.0.1:8090", &[], value), + "127.0.0.1:8090" + ); + } + assert_eq!( + advertised_addr("127.0.0.1:8090", &[], Some("vector.example:8090")), + "vector.example:8090" + ); + } + #[test] fn nprobes_must_be_positive() { let parse = |nprobes: &str| { diff --git a/seaweed-vector/crates/server/src/master_client.rs b/seaweed-vector/crates/server/src/master_client.rs index 7476e992c7..09d58aaf29 100644 --- a/seaweed-vector/crates/server/src/master_client.rs +++ b/seaweed-vector/crates/server/src/master_client.rs @@ -8,16 +8,18 @@ use std::time::Duration; use anyhow::{Context, Result}; use tokio::time::sleep; -use crate::master_pb::{KeepConnectedRequest, seaweed_client::SeaweedClient}; +use crate::master_pb::{ + KeepConnectedRequest, ListClusterNodesRequest, seaweed_client::SeaweedClient, +}; use crate::security::GrpcTls; pub use weed_lance_worker::cluster::{CONNECT_TIMEOUT, connect, grpc_addr}; -fn registration(client_addr: &str) -> KeepConnectedRequest { +fn registration(client_addr: &str, filer_group: &str) -> KeepConnectedRequest { KeepConnectedRequest { client_type: "vector".into(), client_address: client_addr.into(), version: env!("CARGO_PKG_VERSION").into(), - filer_group: String::new(), + filer_group: filer_group.into(), data_center: String::new(), rack: String::new(), leave_lock_ring: false, @@ -39,12 +41,22 @@ fn http_addr(master: &str) -> String { /// Retry forever: try each master until a stream opens, hold it until it /// errors, then rediscover — same shape as wdclient.KeepConnectedToMaster. -pub async fn keep_connected(masters: Vec, client_addr: String, tls: Option) { +pub async fn keep_connected( + masters: Vec, + client_addr: String, + filer_group: String, + tls: Option, + metrics: crate::metrics::Metrics, +) { let client_addr = client_addr.as_str(); + let filer_group = filer_group.as_str(); let tls = tls.as_ref(); + let metrics = &metrics; loop { try_all_masters(&masters, |master| async move { - stream_to(&master, client_addr, tls).await + let result = stream_to(&master, client_addr, filer_group, tls, Some(&metrics)).await; + metrics.set_connected(false).await; + result }) .await; sleep(Duration::from_secs(3)).await; @@ -83,14 +95,16 @@ where async fn stream_to( master: &str, client_addr: &str, + filer_group: &str, tls: Option<&GrpcTls>, + metrics: Option<&crate::metrics::Metrics>, ) -> Result> { let channel = connect(master, tls).await?; // A master answers at once, a follower with its leader hint. let answer_by = tokio::time::Instant::now() + CONNECT_TIMEOUT; let mut client = SeaweedClient::new(channel); let (tx, rx) = tokio::sync::mpsc::channel(8); - tx.send(registration(client_addr)).await?; + tx.send(registration(client_addr, filer_group)).await?; let mut stream = tokio::time::timeout_at( answer_by, client.keep_connected(tokio_stream::wrappers::ReceiverStream::new(rx)), @@ -99,6 +113,8 @@ async fn stream_to( .context("master did not answer keep-connected")?? .into_inner(); let mut answered = false; + let interval = Duration::from_secs(5); + let mut heartbeat = tokio::time::interval_at(tokio::time::Instant::now() + interval, interval); // The master keeps the node live while the stream is open; resend the // registration periodically so a reconnect never leaves a stale entry. loop { @@ -117,23 +133,66 @@ async fn stream_to( return Ok(Some(loc.leader)); } } + if let Some(metrics) = metrics { metrics.set_connected(true).await; } } None => anyhow::bail!("master closed the keep-connected stream"), }, _ = tokio::time::sleep_until(answer_by), if !answered => { anyhow::bail!("master did not answer keep-connected"); } - _ = sleep(Duration::from_secs(5)) => { - tx.send(registration(client_addr)).await?; + _ = heartbeat.tick() => { + tokio::time::timeout(CONNECT_TIMEOUT, tx.send(registration(client_addr, filer_group))).await + .context("registration send stalled")??; + probe_registration(&mut client).await?; } } } } +async fn probe_registration(client: &mut SeaweedClient) -> Result<()> { + tokio::time::timeout( + CONNECT_TIMEOUT, + client.list_cluster_nodes(ListClusterNodesRequest { + client_type: "vector".into(), + ..Default::default() + }), + ) + .await + .context("registration heartbeat stalled")??; + Ok(()) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn registration_preserves_filer_group() { + assert_eq!( + registration("vector:8090", "tenant-a").filer_group, + "tenant-a" + ); + } + + #[tokio::test] + async fn stalled_registration_heartbeat_is_bounded() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let hold = tokio::spawn(async move { + let (_connection, _) = listener.accept().await.unwrap(); + std::future::pending::<()>().await; + }); + let channel = tonic::transport::Endpoint::from_shared(format!("http://{address}")) + .unwrap() + .connect_lazy(); + let mut client = SeaweedClient::new(channel); + let result = tokio::time::timeout(CONNECT_TIMEOUT * 3, probe_registration(&mut client)) + .await + .unwrap(); + assert!(format!("{:#}", result.unwrap_err()).contains("registration heartbeat stalled")); + hold.abort(); + } + /// Runs one pass against scripted masters and returns the dial order. async fn pass(masters: &[&str], answer: fn(&str) -> Result>) -> Vec { let masters: Vec = masters.iter().map(|m| m.to_string()).collect(); @@ -183,7 +242,7 @@ mod tests { #[tokio::test] async fn dial_to_a_black_holed_master_is_bounded() { // TEST-NET-1 (RFC 5737) is reserved, so nothing answers the SYN. - let dial = stream_to("192.0.2.1:9333", "127.0.0.1:8080", None); + let dial = stream_to("192.0.2.1:9333", "127.0.0.1:8080", "", None, None); let result = tokio::time::timeout(CONNECT_TIMEOUT * 3, dial).await; assert!(matches!(result, Ok(Err(_))), "dial was not bounded"); } @@ -197,7 +256,7 @@ mod tests { let _ = tokio::io::copy(&mut conn, &mut tokio::io::sink()).await; }); let master = format!("127.0.0.1:9333.{port}"); - let open = stream_to(&master, "127.0.0.1:8080", None); + let open = stream_to(&master, "127.0.0.1:8080", "", None, None); match tokio::time::timeout(CONNECT_TIMEOUT * 3, open).await { Ok(Err(e)) => assert!(format!("{e:#}").contains("did not answer"), "{e:#}"), other => panic!("open was not bounded: {other:?}"), @@ -232,7 +291,7 @@ mod tests { }); let master = format!("127.0.0.1:9333.{port}"); let tls = credentials(); - let open = stream_to(&master, "127.0.0.1:8080", Some(&tls)); + let open = stream_to(&master, "127.0.0.1:8080", "", Some(&tls), None); match tokio::time::timeout(CONNECT_TIMEOUT * 3, open).await { Ok(Err(e)) => assert!(format!("{e:#}").contains("did not complete"), "{e:#}"), other => panic!("handshake was not bounded: {other:?}"), @@ -258,16 +317,19 @@ mod tests { let mut tls = credentials(); tls.ca_path = fifo.display().to_string(); let master = format!("127.0.0.1:9333.{port}"); - let open = stream_to(&master, "127.0.0.1:8080", Some(&tls)); + let open = stream_to(&master, "127.0.0.1:8080", "", Some(&tls), None); let result = tokio::time::timeout(CONNECT_TIMEOUT * 3, open).await; - // A read+write open of a FIFO does not block, and it lets the - // blocked read finish so the runtime can shut down. - drop( - std::fs::OpenOptions::new() - .read(true) - .write(true) - .open(&fifo), - ); + use std::io::Write; + let mut cleanup = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(&fifo) + .unwrap(); + let replacement = dir.path().join("ca.replacement"); + std::fs::write(&replacement, b"invalid CA").unwrap(); + std::fs::rename(&replacement, &fifo).unwrap(); + cleanup.write_all(b"invalid CA").unwrap(); + drop(cleanup); match result { Ok(Err(e)) => assert!(format!("{e:#}").contains("did not complete"), "{e:#}"), other => panic!("credential read was not bounded: {other:?}"), diff --git a/seaweed-vector/crates/server/src/metrics.rs b/seaweed-vector/crates/server/src/metrics.rs new file mode 100644 index 0000000000..2b0cab082f --- /dev/null +++ b/seaweed-vector/crates/server/src/metrics.rs @@ -0,0 +1,341 @@ +use std::net::SocketAddr; + +use anyhow::Result; +use axum::{Router, http::StatusCode, routing::get}; +use prometheus::{ + Encoder, Histogram, HistogramOpts, HistogramVec, IntCounter, IntCounterVec, IntGauge, + IntGaugeVec, Opts, Registry, TextEncoder, +}; +use tonic_health::{ServingStatus, server::HealthReporter}; + +use crate::read_cache::ReadCache; + +#[derive(Clone)] +pub struct Metrics { + pub registry: Registry, + pub connected: IntGauge, + pub requests: IntCounterVec, + pub request_seconds: HistogramVec, + pub in_flight: IntGaugeVec, + pub query_results: IntCounter, + pub cache_requests: IntCounterVec, + pub storage_bytes: IntCounter, + pub storage_read_seconds: Histogram, + cache_bytes: IntGauge, + cache_entries: IntGauge, + health: Option, +} + +impl Metrics { + pub fn new() -> Result { + let registry = Registry::new(); + let opts = |name: &str, help: &str| { + Opts::new(name, help) + .namespace("SeaweedFS") + .subsystem("vector") + }; + let buckets = prometheus::exponential_buckets(0.0001, 2.0, 24)?; + let histogram = + |name: &str, help: &str| HistogramOpts::from(opts(name, help)).buckets(buckets.clone()); + let connected = IntGauge::with_opts(opts( + "connected", + "Whether the engine is registered with the master.", + ))?; + let requests = IntCounterVec::new( + opts( + "request_total", + "Vector engine requests by operation and gRPC status.", + ), + &["type", "code"], + )?; + let request_seconds = HistogramVec::new( + histogram("request_seconds", "Vector engine request duration."), + &["type"], + )?; + let in_flight = IntGaugeVec::new( + opts("in_flight_requests", "Vector engine requests in progress."), + &["type"], + )?; + let query_results = + IntCounter::with_opts(opts("query_results_total", "Vectors returned by queries."))?; + let cache_requests = IntCounterVec::new( + opts("cache_requests_total", "Immutable reads by cache outcome."), + &["result"], + )?; + let storage_bytes = IntCounter::with_opts(opts( + "storage_read_bytes_total", + "Bytes fetched for immutable cache reads.", + ))?; + let storage_read_seconds = Histogram::with_opts(histogram( + "storage_read_seconds", + "Immutable storage fetch duration.", + ))?; + let cache_bytes = + IntGauge::with_opts(opts("cache_bytes", "Current cache weight in bytes."))?; + let cache_entries = IntGauge::with_opts(opts("cache_entries", "Current cache entries."))?; + for collector in [ + Box::new(connected.clone()) as Box, + Box::new(requests.clone()), + Box::new(request_seconds.clone()), + Box::new(in_flight.clone()), + Box::new(query_results.clone()), + Box::new(cache_requests.clone()), + Box::new(storage_bytes.clone()), + Box::new(storage_read_seconds.clone()), + Box::new(cache_bytes.clone()), + Box::new(cache_entries.clone()), + ] { + registry.register(collector)?; + } + Ok(Self { + registry, + connected, + requests, + request_seconds, + in_flight, + query_results, + cache_requests, + storage_bytes, + storage_read_seconds, + cache_bytes, + cache_entries, + health: None, + }) + } + + pub fn with_health(mut self, health: HealthReporter) -> Self { + self.health = Some(health); + self + } + + pub async fn set_connected(&self, connected: bool) { + self.connected.set(i64::from(connected)); + if let Some(health) = self.health.clone() { + let status = if connected { + ServingStatus::Serving + } else { + ServingStatus::NotServing + }; + health.set_service_status("", status).await; + health + .set_service_status("vector_pb.VectorService", status) + .await; + } + } + + pub fn start(&self, operation: &str) -> InFlight { + let gauge = self.in_flight.with_label_values(&[operation]); + gauge.inc(); + InFlight { + gauge, + requests: self.requests.clone(), + operation: operation.into(), + completed: false, + _timer: self + .request_seconds + .with_label_values(&[operation]) + .start_timer(), + } + } + + fn gather(&self, cache: Option<&ReadCache>) -> Result { + if let Some(cache) = cache { + let (entries, bytes) = cache.size(); + self.cache_entries.set(entries as i64); + self.cache_bytes.set(bytes as i64); + } + let mut buffer = Vec::new(); + TextEncoder::new().encode(&self.registry.gather(), &mut buffer)?; + Ok(String::from_utf8(buffer)?) + } +} + +pub struct InFlight { + gauge: IntGauge, + requests: IntCounterVec, + operation: String, + completed: bool, + _timer: prometheus::HistogramTimer, +} + +impl InFlight { + pub fn complete(&mut self, error: Option) { + let code = error + .map(|c| format!("{c:?}")) + .unwrap_or_else(|| "OK".into()); + self.requests + .with_label_values(&[&self.operation, &code]) + .inc(); + self.completed = true; + } +} + +impl Drop for InFlight { + fn drop(&mut self) { + self.gauge.dec(); + if !self.completed { + self.requests + .with_label_values(&[&self.operation, "Cancelled"]) + .inc(); + } + } +} + +pub async fn serve(metrics: Metrics, address: SocketAddr, cache: Option) -> Result<()> { + let listener = tokio::net::TcpListener::bind(address).await?; + serve_listener(metrics, listener, cache).await +} + +pub async fn serve_listener( + metrics: Metrics, + listener: tokio::net::TcpListener, + cache: Option, +) -> Result<()> { + let readiness = metrics.clone(); + let app = Router::new() + .route("/health", get(|| async { StatusCode::OK })) + .route( + "/ready", + get(move || { + let metrics = readiness.clone(); + async move { + if metrics.connected.get() == 1 { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + } + } + }), + ) + .route( + "/metrics", + get(move || { + let metrics = metrics.clone(); + let cache = cache.clone(); + async move { + metrics + .gather(cache.as_ref()) + .map(|body| ([("content-type", "text/plain; version=0.0.4")], body)) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) + } + }), + ); + axum::serve(listener, app).await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn endpoints_follow_registration_and_publish_metrics() { + let metrics = Metrics::new().unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(serve_listener(metrics.clone(), listener, None)); + let client = reqwest::Client::new(); + assert_eq!( + client + .get(format!("{endpoint}/health")) + .send() + .await + .unwrap() + .status(), + 200 + ); + assert_eq!( + client + .get(format!("{endpoint}/ready")) + .send() + .await + .unwrap() + .status(), + 503 + ); + metrics.set_connected(true).await; + assert_eq!( + client + .get(format!("{endpoint}/ready")) + .send() + .await + .unwrap() + .status(), + 200 + ); + let in_flight = metrics.start("QueryVectors"); + let body = client + .get(format!("{endpoint}/metrics")) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + assert!(body.contains("SeaweedFS_vector_connected 1")); + assert!(body.contains("SeaweedFS_vector_in_flight_requests{type=\"QueryVectors\"} 1")); + drop(in_flight); + assert_eq!( + metrics.in_flight.with_label_values(&["QueryVectors"]).get(), + 0 + ); + metrics.set_connected(false).await; + assert_eq!( + client + .get(format!("{endpoint}/ready")) + .send() + .await + .unwrap() + .status(), + 503 + ); + task.abort(); + } + + #[tokio::test] + async fn grpc_health_follows_registration() { + use tonic_health::pb::{HealthCheckRequest, health_client::HealthClient}; + let (reporter, service) = tonic_health::server::health_reporter(); + let metrics = Metrics::new().unwrap().with_health(reporter); + metrics.set_connected(false).await; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn( + tonic::transport::Server::builder() + .add_service(service) + .serve_with_incoming(tokio_stream::wrappers::TcpListenerStream::new(listener)), + ); + let channel = tonic::transport::Endpoint::from_shared(endpoint) + .unwrap() + .connect() + .await + .unwrap(); + let mut client = HealthClient::new(channel); + let request = || HealthCheckRequest { + service: "vector_pb.VectorService".into(), + }; + assert_eq!( + client.check(request()).await.unwrap().into_inner().status, + 2 + ); + metrics.set_connected(true).await; + assert_eq!( + client.check(request()).await.unwrap().into_inner().status, + 1 + ); + metrics.set_connected(false).await; + assert_eq!( + client.check(request()).await.unwrap().into_inner().status, + 2 + ); + task.abort(); + } +} + +impl std::fmt::Debug for Metrics { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Metrics") + .field("connected", &self.connected.get()) + .finish() + } +} diff --git a/seaweed-vector/crates/server/src/observed.rs b/seaweed-vector/crates/server/src/observed.rs new file mode 100644 index 0000000000..55aa563efd --- /dev/null +++ b/seaweed-vector/crates/server/src/observed.rs @@ -0,0 +1,122 @@ +use crate::metrics::Metrics; +use crate::pb::vector::{self as pb, vector_service_server::VectorService}; +use crate::service::VectorServer; +use tonic::{Request, Response, Status}; + +pub struct ObservedService { + pub inner: VectorServer, + pub metrics: Metrics, +} + +#[tonic::async_trait] +impl VectorService for ObservedService { + async fn create_index( + &self, + request: Request, + ) -> Result, Status> { + let mut in_flight = self.metrics.start("CreateIndex"); + let result = self.inner.create_index(request).await; + in_flight.complete(result.as_ref().err().map(|e| e.code())); + result + } + async fn drop_index( + &self, + request: Request, + ) -> Result, Status> { + let mut in_flight = self.metrics.start("DropIndex"); + let result = self.inner.drop_index(request).await; + in_flight.complete(result.as_ref().err().map(|e| e.code())); + result + } + async fn put_vectors( + &self, + request: Request, + ) -> Result, Status> { + let mut in_flight = self.metrics.start("PutVectors"); + let result = self.inner.put_vectors(request).await; + in_flight.complete(result.as_ref().err().map(|e| e.code())); + result + } + async fn get_vectors( + &self, + request: Request, + ) -> Result, Status> { + let mut in_flight = self.metrics.start("GetVectors"); + let result = self.inner.get_vectors(request).await; + in_flight.complete(result.as_ref().err().map(|e| e.code())); + result + } + async fn delete_vectors( + &self, + request: Request, + ) -> Result, Status> { + let mut in_flight = self.metrics.start("DeleteVectors"); + let result = self.inner.delete_vectors(request).await; + in_flight.complete(result.as_ref().err().map(|e| e.code())); + result + } + async fn list_vectors( + &self, + request: Request, + ) -> Result, Status> { + let mut in_flight = self.metrics.start("ListVectors"); + let result = self.inner.list_vectors(request).await; + in_flight.complete(result.as_ref().err().map(|e| e.code())); + result + } + async fn query_vectors( + &self, + request: Request, + ) -> Result, Status> { + let mut in_flight = self.metrics.start("QueryVectors"); + let result = self.inner.query_vectors(request).await; + in_flight.complete(result.as_ref().err().map(|e| e.code())); + if let Ok(response) = &result { + self.metrics + .query_results + .inc_by(response.get_ref().vectors.len() as u64); + } + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn failures_are_counted_and_release_in_flight_requests() { + let service = ObservedService { + inner: VectorServer::new(), + metrics: Metrics::new().unwrap(), + }; + let response = service + .query_vectors(Request::new(pb::QueryVectorsRequest::default())) + .await; + let code = format!("{:?}", response.unwrap_err().code()); + assert_eq!( + service + .metrics + .requests + .with_label_values(&["QueryVectors", &code]) + .get(), + 1 + ); + assert_eq!( + service + .metrics + .in_flight + .with_label_values(&["QueryVectors"]) + .get(), + 0 + ); + assert_eq!( + service + .metrics + .request_seconds + .with_label_values(&["QueryVectors"]) + .get_sample_count(), + 1 + ); + } +} diff --git a/seaweed-vector/crates/server/src/read_cache.rs b/seaweed-vector/crates/server/src/read_cache.rs index ec6f21bc2b..71e881714f 100644 --- a/seaweed-vector/crates/server/src/read_cache.rs +++ b/seaweed-vector/crates/server/src/read_cache.rs @@ -9,6 +9,7 @@ use std::fmt; use std::ops::Range; use std::sync::Arc; +use crate::metrics::Metrics; use bytes::Bytes; use futures::StreamExt; use futures::stream::BoxStream; @@ -42,6 +43,7 @@ pub struct ReadCache { /// Keeps entries apart from those of another [`Self::scoped`] view, for /// stores that share a prefix but not their bytes. scope: Arc, + metrics: Option, } impl ReadCache { @@ -54,14 +56,26 @@ impl ReadCache { }) .build(), scope: "".into(), + metrics: None, } } + pub fn with_metrics(mut self, metrics: Metrics) -> Self { + self.metrics = Some(metrics); + self + } + + pub fn size(&self) -> (u64, u64) { + self.cache.run_pending_tasks(); + (self.cache.entry_count(), self.cache.weighted_size()) + } + /// The same cache, keyed apart under `scope`. pub fn scoped(&self, scope: &str) -> Self { Self { cache: self.cache.clone(), scope: scope.into(), + metrics: self.metrics.clone(), } } } @@ -81,6 +95,7 @@ impl WrappingObjectStore for ReadCache { inner: original, store: format!("{}|{store_prefix}", self.scope).into(), cache: self.cache.clone(), + metrics: self.metrics.clone(), }) } } @@ -136,6 +151,7 @@ struct CachedStore { inner: Arc, store: Arc, cache: moka::sync::Cache, + metrics: Option, } impl CachedStore { @@ -184,14 +200,47 @@ impl ObjectStore for CachedStore { object: Some(object), }) = self.cache.get(&key) { + if let Some(metrics) = &self.metrics { + metrics.cache_requests.with_label_values(&["hit"]).inc(); + } return Ok(whole(bytes, &object)); } + if let Some(metrics) = &self.metrics { + metrics.cache_requests.with_label_values(&["miss"]).inc(); + } + let _timer = self + .metrics + .as_ref() + .map(|m| m.storage_read_seconds.start_timer()); let result = self.inner.get_opts(location, options).await?; if result.meta.size > MAX_WHOLE_OBJECT { - return Ok(result); + let meta = result.meta.clone(); + let range = result.range.clone(); + let attributes = result.attributes.clone(); + let metrics = self.metrics.clone(); + let timer = _timer; + let stream = result + .into_stream() + .map(move |chunk| { + let _timer = &timer; + if let (Some(metrics), Ok(bytes)) = (&metrics, &chunk) { + metrics.storage_bytes.inc_by(bytes.len() as u64); + } + chunk + }) + .boxed(); + return Ok(GetResult { + payload: GetResultPayload::Stream(stream), + meta, + range, + attributes, + }); } let object = Arc::new((result.meta.clone(), result.attributes.clone())); let bytes = owned(result.bytes().await?); + if let Some(metrics) = &self.metrics { + metrics.storage_bytes.inc_by(bytes.len() as u64); + } self.cache.insert( key, Entry { @@ -220,7 +269,21 @@ impl ObjectStore for CachedStore { .filter(|(_, hit)| hit.is_none()) .map(|(range, _)| range.clone()) .collect(); + if let Some(metrics) = &self.metrics { + metrics + .cache_requests + .with_label_values(&["miss"]) + .inc_by(missing.len() as u64); + metrics + .cache_requests + .with_label_values(&["hit"]) + .inc_by((ranges.len() - missing.len()) as u64); + } if !missing.is_empty() { + let _timer = self + .metrics + .as_ref() + .map(|m| m.storage_read_seconds.start_timer()); let mut fetched = self.inner.get_ranges(location, &missing).await?.into_iter(); for (range, slot) in ranges.iter().zip(out.iter_mut()) { if slot.is_some() { @@ -230,6 +293,9 @@ impl ObjectStore for CachedStore { break; }; let bytes = owned(bytes); + if let Some(metrics) = &self.metrics { + metrics.storage_bytes.inc_by(bytes.len() as u64); + } self.cache.insert( self.key(location, Some(range.clone())), Entry { @@ -491,6 +557,16 @@ mod tests { }; create(&server, &table, 4).await; + create(&server, &table, 4).await; + let mismatch = server + .create_index(Request::new(pb::CreateIndexRequest { + table: Some(table.clone()), + dimension: 3, + ..Default::default() + })) + .await + .unwrap_err(); + assert_eq!(mismatch.code(), tonic::Code::AlreadyExists); put( &server, &table, diff --git a/seaweed-vector/crates/server/src/service.rs b/seaweed-vector/crates/server/src/service.rs index d9ec782793..df7c9ae0e9 100644 --- a/seaweed-vector/crates/server/src/service.rs +++ b/seaweed-vector/crates/server/src/service.rs @@ -95,6 +95,44 @@ fn schema_for( ])) } +fn matches_index_schema( + dataset: &Dataset, + dimension: usize, + metric: pb::DistanceMetric, + non_filterable: &[String], +) -> bool { + let Ok(layout) = VectorTable::resolve(dataset, Some(VECTOR_COLUMN), Some(KEY_COLUMN)) else { + return false; + }; + let schema = Schema::from(dataset.schema()); + let Ok(vector) = schema.field_with_name(VECTOR_COLUMN) else { + return false; + }; + let Ok(metadata) = schema.field_with_name(METADATA_COLUMN) else { + return false; + }; + let stored_keys = metadata + .metadata() + .get("non_filterable_metadata_keys") + .map(|keys| serde_json::from_str::>(keys)) + .transpose(); + let Ok(stored_keys) = stored_keys else { + return false; + }; + let mut stored_keys = stored_keys.unwrap_or_default(); + let mut requested_keys = non_filterable.to_vec(); + stored_keys.sort(); + requested_keys.sort(); + layout.dimension == dimension + && vector.metadata().get("distance_metric").map(String::as_str) + == Some(metric.as_str_name()) + && stored_keys == requested_keys + && metadata + .metadata() + .get("ARROW:extension:name") + .is_some_and(|name| name == "lance.json") +} + fn batch_of(vectors: &[pb::Vector], dimension: usize, schema: Arc) -> Result { let mut seen = std::collections::HashSet::new(); for vector in vectors { @@ -536,8 +574,28 @@ impl VectorService for VectorServer { ..Default::default() }), ) - .await - .map_err(internal)?; + .await; + let dataset = match dataset { + Ok(dataset) => dataset, + Err(error) => match self.open(&table).await { + Ok(existing) + if matches_index_schema( + &existing, + dimension, + metric, + &req.non_filterable_metadata_keys, + ) => + { + existing + } + Ok(_) => { + return Err(Status::already_exists( + "index exists with a different schema", + )); + } + Err(_) => return Err(internal(error)), + }, + }; Ok(Response::new(pb::CreateIndexResponse { version: dataset.version().version, })) diff --git a/seaweed-vector/crates/server/tests/cache_metrics.rs b/seaweed-vector/crates/server/tests/cache_metrics.rs new file mode 100644 index 0000000000..7b63f4d2a3 --- /dev/null +++ b/seaweed-vector/crates/server/tests/cache_metrics.rs @@ -0,0 +1,111 @@ +use std::sync::Arc; + +use bytes::Bytes; +use lance::io::WrappingObjectStore; +use object_store::{ObjectStoreExt, memory::InMemory, path::Path}; +use weed_vector_server::{metrics::Metrics, read_cache::ReadCache}; + +#[tokio::test] +async fn immutable_reads_report_hits_misses_bytes_and_weight() { + let metrics = Metrics::new().unwrap(); + let cache = ReadCache::new(1024).with_metrics(metrics.clone()); + let store = cache.wrap("memory", Arc::new(InMemory::new())); + let path = Path::from("data/example.lance"); + store + .put(&path, Bytes::from_static(b"abc").into()) + .await + .unwrap(); + for _ in 0..2 { + assert_eq!( + store.get(&path).await.unwrap().bytes().await.unwrap(), + "abc" + ); + } + assert_eq!(metrics.cache_requests.with_label_values(&["miss"]).get(), 1); + assert_eq!(metrics.cache_requests.with_label_values(&["hit"]).get(), 1); + assert_eq!(metrics.storage_bytes.get(), 3); + assert_eq!(metrics.storage_read_seconds.get_sample_count(), 1); + let (entries, weight) = cache.size(); + assert_eq!(entries, 1); + assert!(weight >= 3); + let metrics_endpoint = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}/metrics", metrics_endpoint.local_addr().unwrap()); + let task = tokio::spawn(weed_vector_server::metrics::serve_listener( + metrics, + metrics_endpoint, + Some(cache), + )); + let body = reqwest::get(endpoint).await.unwrap().text().await.unwrap(); + assert!(body.contains("SeaweedFS_vector_cache_entries 1")); + assert!(body.contains("SeaweedFS_vector_cache_bytes ")); + task.abort(); +} + +#[tokio::test] +async fn cancelled_requests_are_counted_once() { + let metrics = Metrics::new().unwrap(); + let guard = metrics.start("QueryVectors"); + let task = tokio::spawn(async move { + let _guard = guard; + std::future::pending::<()>().await; + }); + task.abort(); + let _ = task.await; + assert_eq!( + metrics.in_flight.with_label_values(&["QueryVectors"]).get(), + 0 + ); + assert_eq!( + metrics + .requests + .with_label_values(&["QueryVectors", "Cancelled"]) + .get(), + 1 + ); + assert_eq!( + metrics + .request_seconds + .with_label_values(&["QueryVectors"]) + .get_sample_count(), + 1 + ); + { + let mut guard = metrics.start("QueryVectors"); + guard.complete(None); + } + assert_eq!( + metrics + .requests + .with_label_values(&["QueryVectors", "OK"]) + .get(), + 1 + ); + assert_eq!( + metrics + .requests + .with_label_values(&["QueryVectors", "Cancelled"]) + .get(), + 1 + ); +} + +#[tokio::test] +async fn large_immutable_reads_report_bytes_without_caching() { + let metrics = Metrics::new().unwrap(); + let cache = ReadCache::new(16 * 1024 * 1024).with_metrics(metrics.clone()); + let store = cache.wrap("memory", Arc::new(InMemory::new())); + let path = Path::from("data/large.lance"); + let size = 5 * 1024 * 1024; + store + .put(&path, Bytes::from(vec![0; size]).into()) + .await + .unwrap(); + let result = store.get(&path).await.unwrap(); + assert_eq!(metrics.storage_bytes.get(), 0); + assert_eq!(metrics.storage_read_seconds.get_sample_count(), 0); + assert_eq!(result.bytes().await.unwrap().len(), size); + assert_eq!(metrics.storage_bytes.get(), size as u64); + assert_eq!(metrics.cache_requests.with_label_values(&["miss"]).get(), 1); + assert_eq!(metrics.storage_read_seconds.get_sample_count(), 1); + assert_eq!(cache.size(), (0, 0)); +} diff --git a/test/s3vectors/conftest.py b/test/s3vectors/conftest.py index 0311d242d5..a5eab9f19b 100644 --- a/test/s3vectors/conftest.py +++ b/test/s3vectors/conftest.py @@ -123,7 +123,7 @@ def write_tls(directory): "-out", "outsider.pem", "-days", "2", "-extfile", "ext.cnf") cert, key, ca = (f'{key} = {json.dumps(str(directory / name), ensure_ascii=False)}\n' for key, name in (("cert", "cert.pem"), ("key", "key.pem"), ("ca", "ca.pem"))) pem = cert + key + ca - grpc = "".join(f"[grpc.{component}]\n{cert}{key}\n" for component in ("master", "volume", "filer", "s3")) + grpc = "".join(f"[grpc.{component}]\n{cert}{key}\n" for component in ("master", "volume", "filer", "s3", "admin", "worker")) # Clients (the gateway, the engine) accept only servers named localhost; # the engine serves only clients named localhost. grpc += f"[grpc.client]\n{cert}{key}allowed_server_commonNames = \"localhost\"\n" @@ -248,12 +248,12 @@ def run_cluster(param, tmp_path_factory, identities): with contextlib.ExitStack() as stack: with contextlib.ExitStack() as reservations: ports = [] - for _ in range(14): + for _ in range(15): sock = reservations.enter_context(socket.socket()) sock.bind(("127.0.0.1", 0)) ports.append(sock.getsockname()[1]) (master, master_grpc, vol, vol_grpc, filer, filer_grpc, s3, s3_grpc, lance, vector, worker1, worker2, - admin, admin_grpc) = ports + admin, admin_grpc, vector_metrics) = ports master_addr = f"127.0.0.1:{master}.{master_grpc}" filer_addr = f"127.0.0.1:{filer}.{filer_grpc}" @@ -304,10 +304,11 @@ def run_cluster(param, tmp_path_factory, identities): # VECTOR_ENCRYPT_VOLUMES=1 turns on the filer's volume encryption, which # the gateway, the engine and the worker all follow: every chunk they # store is encrypted, and direct reads must undo it. + filer_group = os.environ.get("VECTOR_FILER_GROUP", "") encrypt = ["-encryptVolumeData"] if os.environ.get("VECTOR_ENCRYPT_VOLUMES") else [] start("filer", [weed, "filer", f"-master={master_addr}", "-ip=127.0.0.1", "-ip.bind=127.0.0.1", f"-port={filer}", f"-port.grpc={filer_grpc}", - f"-defaultStoreDir={directory / 'filer'}", *encrypt]) + f"-defaultStoreDir={directory / 'filer'}", f"-filerGroup={filer_group}", *encrypt]) wait_for(lambda: listening(filer_grpc), processes, "filer not ready") endpoint = f"http://127.0.0.1:{s3}" # weed s3 turns AWS_ACCESS_KEY_ID into an admin identity. @@ -325,8 +326,12 @@ def run_cluster(param, tmp_path_factory, identities): # The engine keeps tables on the filers and volume servers itself; it # has no S3 credentials to be given, and encrypts its chunks for the # volume servers as the gateway does. - start("vector", [engine, "--masters", master_addr, "--listen", f"127.0.0.1:{vector}"]) + start("vector", [engine, "--masters", master_addr, "--listen", f"127.0.0.1:{vector}", "--metrics-port", str(vector_metrics), "--filer-group", filer_group]) wait_for(lambda: listening(vector), processes, "vector engine not ready") + def vector_ready(): + with urllib.request.urlopen(f"http://127.0.0.1:{vector_metrics}/ready", timeout=1) as response: + return response.status == 200 + wait_for(vector_ready, processes, "vector engine did not register with master") if os.environ.get("VECTOR_TLS"): wait_for(lambda: listening(s3_grpc), processes, "s3 gRPC not ready") for name, port in (("master", master_grpc), ("volume", vol_grpc), ("filer", filer_grpc), @@ -335,7 +340,7 @@ def run_cluster(param, tmp_path_factory, identities): probe_grpc_tls("engine", vector, directory, outsider=True) wait_for( - lambda: f"Cluster: vector node 127.0.0.1:{vector} added to group ''" + lambda: f"Cluster: vector node 127.0.0.1:{vector} added to group '{filer_group}'" in (directory / "master.log").read_text(), processes, "vector engine never registered with the master", @@ -358,18 +363,22 @@ def run_cluster(param, tmp_path_factory, identities): if lance_worker: (directory / "admin").mkdir() admin_url = f"http://127.0.0.1:{admin}" - start("admin", [weed, "admin", "-ip=127.0.0.1", f"-port={admin}", f"-port.grpc={admin_grpc}", + start("admin", [weed, "admin", f"-filerGroup={filer_group}", "-ip=127.0.0.1", f"-port={admin}", f"-port.grpc={admin_grpc}", "-port.legacy=0", f"-master={master_addr}", f"-dataDir={directory / 'admin'}"]) wait_for(lambda: http_ready(f"{admin_url}/api/plugin/job-types"), processes, "admin not ready") + tls_args = [] + if os.environ.get("VECTOR_TLS"): + tls_args = ["--tls-ca", str(directory / "ca.pem"), "--tls-cert", str(directory / "cert.pem"), "--tls-key", str(directory / "key.pem"), "--tls-server-name", "localhost"] start("lance-worker", [lance_worker, "--admin", f"127.0.0.1:{admin}.{admin_grpc}", "--namespace", f"http://127.0.0.1:{lance}", "--id", "vectors-test-worker", - "--masters", master_addr]) + "--masters", master_addr, "--filer-group", filer_group, *tls_args]) wait_for(lambda: "lance_vector_index" in [jt["job_type"] for jt in json.load(urllib.request.urlopen(f"{admin_url}/api/plugin/job-types", timeout=2))], processes, "the Lance worker never registered with admin") yield {"endpoint": endpoint, "mode": mode, "processes": processes, "directory": directory, "filer": filer, "masters": master_addr, "filer_signing_key": filer_signing_key, + "vector_metrics": f"http://127.0.0.1:{vector_metrics}", "catalog": f"http://127.0.0.1:{lance}", "admin": admin_url, "env": process_env()} diff --git a/test/s3vectors/s3.json b/test/s3vectors/s3.json index 436bd30be9..1e3851e215 100644 --- a/test/s3vectors/s3.json +++ b/test/s3vectors/s3.json @@ -39,6 +39,24 @@ "actions": [ "Write" ] + }, + { + "name": "policy-reader", + "credentials": [ + { + "accessKey": "policy-reader", + "secretKey": "policy-reader-secret" + } + ], + "policyNames": [ + "vector-policy" + ] + } + ], + "policies": [ + { + "name": "vector-policy", + "content": "{\"Version\": \"2012-10-17\", \"Statement\": [{\"Effect\": \"Allow\", \"Action\": [\"s3vectors:GetVectors\", \"s3vectors:QueryVectors\"], \"Resource\": \"arn:aws:s3vectors:*:*:bucket/vector-iam/index/docs\"}, {\"Effect\": \"Allow\", \"Action\": \"s3vectors:CreateVectorBucket\", \"Resource\": \"*\"}, {\"Effect\": \"Allow\", \"Action\": \"s3vectors:TagResource\", \"Resource\": \"arn:aws:s3vectors:*:*:bucket/vector-iam-*\"}, {\"Effect\": \"Deny\", \"Action\": \"s3vectors:QueryVectors\", \"Resource\": \"*\", \"Condition\": {\"StringEquals\": {\"aws:ResourceTag/classification\": \"private\"}}}, {\"Effect\": \"Deny\", \"Action\": \"s3vectors:QueryVectors\", \"Resource\": \"*\", \"Condition\": {\"StringEquals\": {\"s3vectors:VectorBucketTag/environment\": \"production\"}}}, {\"Effect\": \"Deny\", \"Action\": \"s3vectors:CreateVectorBucket\", \"Resource\": \"*\", \"Condition\": {\"StringEquals\": {\"aws:RequestTag/classification\": \"forbidden\"}}}, {\"Effect\": \"Deny\", \"Action\": \"s3vectors:CreateVectorBucket\", \"Resource\": \"*\", \"Condition\": {\"ForAnyValue:StringEquals\": {\"aws:TagKeys\": \"forbidden-key\"}}}]}" } ] } diff --git a/test/s3vectors/s3vectors_test.py b/test/s3vectors/s3vectors_test.py index feda356895..f0640c40b9 100644 --- a/test/s3vectors/s3vectors_test.py +++ b/test/s3vectors/s3vectors_test.py @@ -1,3 +1,4 @@ +import urllib.request """Run the S3 Vectors API against Go/Rust volumes and single/distributed queries.""" import contextlib @@ -150,7 +151,7 @@ def run_cli(cluster, *args): from pathlib import Path cli = Path(os.environ.get("VECTOR_CLI_BINARY", Path(__file__).resolve().parent / "weed-vector-cli")).resolve() command = [str(cli), "--namespace", cluster["catalog"], "--masters", cluster["masters"], - "--filer-signing-key", cluster["filer_signing_key"], *args] + "--filer-signing-key", cluster["filer_signing_key"], "--filer-group", os.environ.get("VECTOR_FILER_GROUP", ""), *args] return json.loads(subprocess.run(command, check=True, capture_output=True, text=True, cwd=cluster["directory"], env=cluster["env"]).stdout) @@ -711,6 +712,43 @@ def test_vector_index_job(vectors, cluster): assert found >= 180, f"recall after retrain {found}/200" +def test_attached_iam_vector_policies(vectors, cluster): + client = boto3.client("s3vectors", endpoint_url=cluster["endpoint"], region_name="us-east-1", + aws_access_key_id="policy-reader", aws_secret_access_key="policy-reader-secret", + config=vectors.meta.config) + bucket = "vector-iam" + vectors.create_vector_bucket(vectorBucketName=bucket) + target = {"vectorBucketName": bucket, "indexName": "docs"} + vectors.create_index(**target, dataType="float32", dimension=3, distanceMetric="cosine") + index_arn = vectors.get_index(**target)["index"]["indexArn"] + bucket_arn = vectors.get_vector_bucket(vectorBucketName=bucket)["vectorBucket"]["vectorBucketArn"] + query = {**target, "queryVector": {"float32": [1, 0, 0]}, "topK": 1} + try: + assert client.query_vectors(**query)["vectors"] == [] + assert client.get_vectors(**target, keys=["missing"])["vectors"] == [] + expect_error("AccessDeniedException", client.put_vectors, **target, + vectors=[{"key": "blocked", "data": {"float32": [1, 0, 0]}}]) + vectors.tag_resource(resourceArn=index_arn, tags={"classification": "private"}) + expect_error("AccessDeniedException", client.query_vectors, **query) + vectors.untag_resource(resourceArn=index_arn, tagKeys=["classification"]) + assert client.query_vectors(**query)["vectors"] == [] + vectors.tag_resource(resourceArn=bucket_arn, tags={"environment": "production"}) + expect_error("AccessDeniedException", client.query_vectors, **query) + vectors.untag_resource(resourceArn=bucket_arn, tagKeys=["environment"]) + assert client.query_vectors(**query)["vectors"] == [] + vectors.put_vector_bucket_policy(vectorBucketName=bucket, policy=json.dumps({"Version": "2012-10-17", "Statement": [ + {"Effect": "Deny", "Principal": "*", "Action": "s3vectors:QueryVectors", "Resource": index_arn}]})) + expect_error("AccessDeniedException", client.query_vectors, **query) + expect_error("AccessDeniedException", client.create_vector_bucket, + vectorBucketName="vector-iam-rejected", tags={"classification": "forbidden"}) + expect_error("AccessDeniedException", client.create_vector_bucket, + vectorBucketName="vector-iam-key-denied", tags={"forbidden-key": "value"}) + finally: + vectors.delete_vector_bucket_policy(vectorBucketName=bucket) + vectors.delete_index(**target) + vectors.delete_vector_bucket(vectorBucketName=bucket) + + def test_invalid_credentials(cluster): client = boto3.client("s3vectors", endpoint_url=cluster["endpoint"], aws_access_key_id="unknown", aws_secret_access_key="unknown", region_name="us-east-1") @@ -719,6 +757,18 @@ def test_invalid_credentials(cluster): assert exc.value.response["ResponseMetadata"]["HTTPStatusCode"] == 403 +def test_vector_engine_observability(vectors, index, cluster): + vectors.query_vectors(**index, queryVector={"float32": [1, 0, 0, 0]}, topK=1) + with urllib.request.urlopen(cluster["vector_metrics"] + "/ready", timeout=5) as response: + assert response.status == 200 + with urllib.request.urlopen(cluster["vector_metrics"] + "/metrics", timeout=5) as response: + metrics = response.read().decode() + assert "SeaweedFS_vector_connected 1" in metrics + assert any(line.startswith("SeaweedFS_vector_request_total{") and 'type="QueryVectors"' in line + and 'code="OK"' in line for line in metrics.splitlines()) + assert "SeaweedFS_vector_request_seconds_bucket" in metrics + + def test_query_without_writer(vectors, index, cluster): from conftest import stop @@ -753,3 +803,5 @@ def test_engine_without_credentials(open_cluster): hits = client.query_vectors(**index, queryVector={"float32": [1, 0]}, topK=1)["vectors"] assert [h["key"] for h in hits] == ["a"] + + diff --git a/weed/admin/dash/admin_data.go b/weed/admin/dash/admin_data.go index 3907e5b236..499de6e154 100644 --- a/weed/admin/dash/admin_data.go +++ b/weed/admin/dash/admin_data.go @@ -32,6 +32,7 @@ type AdminData struct { VolumeServers []VolumeServer `json:"volume_servers"` FilerNodes []FilerNode `json:"filer_nodes"` MessageBrokers []MessageBrokerNode `json:"message_brokers"` + VectorNodes []VectorNode `json:"vector_nodes"` S3Nodes []S3Node `json:"s3_nodes"` DataCenters []DataCenter `json:"datacenters"` LastUpdated time.Time `json:"last_updated"` @@ -250,6 +251,7 @@ func (s *AdminServer) GetAdminData(username string) (AdminData, error) { FilerNodes: filerNodes, MessageBrokers: messageBrokers, S3Nodes: s3Nodes, + VectorNodes: s.getVectorNodesStatus(context.Background()).Engines, DataCenters: topology.DataCenters, LastUpdated: topology.UpdatedAt, TotalEcVolumes: totalEcVolumes, diff --git a/weed/admin/dash/admin_server_filergroup_test.go b/weed/admin/dash/admin_server_filergroup_test.go index 411f8429fa..81d39d2381 100644 --- a/weed/admin/dash/admin_server_filergroup_test.go +++ b/weed/admin/dash/admin_server_filergroup_test.go @@ -28,3 +28,11 @@ func TestListClusterNodesRequestKeepsDefaultFilerGroupEmpty(t *testing.T) { t.Fatalf("FilerGroup = %q, want empty", req.FilerGroup) } } + +func TestVectorDiscoveryUsesRegistrationGroup(t *testing.T) { + s := &AdminServer{filerGroup: "tenant-a"} + request := s.listClusterNodesRequest(cluster.VectorType) + if request.FilerGroup != s.filerGroup { + t.Fatalf("vector registration group must match the filer: %+v", request) + } +} diff --git a/weed/admin/dash/s3vectors_management.go b/weed/admin/dash/s3vectors_management.go index 58204fd2bb..d3055d3e52 100644 --- a/weed/admin/dash/s3vectors_management.go +++ b/weed/admin/dash/s3vectors_management.go @@ -2,11 +2,13 @@ package dash import ( "context" + "encoding/json" "errors" "net/http" "time" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/lance" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" @@ -15,6 +17,7 @@ import ( type S3VectorsBucketsData struct { Username string `json:"username"` Buckets []S3VectorsBucketSummary `json:"buckets"` + Operations VectorOperationsData `json:"operations"` TotalBuckets int `json:"total_buckets"` Truncated bool `json:"truncated,omitempty"` LastUpdated time.Time `json:"last_updated"` @@ -33,6 +36,7 @@ type S3VectorsBucketData struct { Bucket S3VectorsBucketSummary `json:"bucket"` Policy string `json:"policy,omitempty"` Indexes []*s3vectors.Index `json:"indexes"` + Operations VectorOperationsData `json:"operations"` TotalIndexes int `json:"total_indexes"` Truncated bool `json:"truncated,omitempty"` LastUpdated time.Time `json:"last_updated"` @@ -46,11 +50,15 @@ func (c s3VectorsFilerClient) WithFilerClient(_ bool, fn func(client filer_pb.Se return c.s.WithFilerClient(fn) } -// newS3VectorsServer serves the catalog side of S3 Vectors for the admin -// console. It has no vector engine, so it never runs an operation that needs -// one, such as CreateIndex. func newS3VectorsServer(s *AdminServer) *s3vectors.Server { - server := s3vectors.NewServer(s3VectorsFilerClient{s}, nil, nil) + client := s3VectorsFilerClient{s} + backend := s3vectors.NewVectorDiscovery(s.masterClient.ListCachedMasters, s.grpcDialOption, s.filerGroup) + server := s3vectors.NewServer(client, nil, backend) + catalog := lance.NewServer(client, nil) + server.SetDescribe(func(ctx context.Context, id []string) (string, map[string]string, error) { + request := (&http.Request{Method: http.MethodPost, Header: make(http.Header)}).WithContext(ctx) + return catalog.DescribeTableInternal(request, id) + }) server.SetRegion(s3tables.DefaultRegion) server.SetAccountID(s3tables.DefaultAccountID) return server @@ -103,6 +111,7 @@ func (s *AdminServer) GetS3VectorsBucketsData(ctx context.Context) (S3VectorsBuc } req.NextToken = resp.NextToken } + data.Operations = s.GetVectorOperationsData(ctx) data.TotalBuckets = len(data.Buckets) data.LastUpdated = time.Now() return data, nil @@ -148,6 +157,7 @@ func (s *AdminServer) GetS3VectorsBucketData(ctx context.Context, bucket string) } req.NextToken = resp.NextToken } + data.Operations = s.GetVectorOperationsData(ctx) data.TotalIndexes = len(data.Indexes) return data, nil } @@ -166,14 +176,20 @@ func (s *AdminServer) CreateS3VectorsBucket(w http.ResponseWriter, r *http.Reque return } var req struct { - Name string `json:"name"` + Name string `json:"name"` + EncryptionConfiguration map[string]interface{} `json:"encryptionConfiguration,omitempty"` + Tags map[string]string `json:"tags,omitempty"` } if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil { writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error()) return } + if req.EncryptionConfiguration["sseType"] == "AES256" && !s.vectorEncryptionConfigured(r.Context()) { + writeJSONError(w, http.StatusBadRequest, "Cannot verify SSE-S3 support on the S3 gateway; configure s3.public_endpoint and an SSE-S3 key on the gateway and vector engines") + return + } var resp s3vectors.CreateVectorBucketResponse - if err := s.executeS3VectorsOperation(r.Context(), "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: req.Name}, &resp); err != nil { + if err := s.executeS3VectorsOperation(r.Context(), "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: req.Name, EncryptionConfiguration: req.EncryptionConfiguration, Tags: req.Tags}, &resp); err != nil { writeS3VectorsError(w, err) return } @@ -252,3 +268,40 @@ func writeS3VectorsError(w http.ResponseWriter, err error) { } writeJSONError(w, status, e.Message) } + +func (s *AdminServer) ExecuteS3VectorsOperationAPI(w http.ResponseWriter, r *http.Request) { + if !requireSessionCSRFToken(w, r) { + return + } + var req struct { + Operation string `json:"operation"` + Request json.RawMessage `json:"request"` + } + if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil { + writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error()) + return + } + switch req.Operation { + case "CreateIndex", "PutVectorBucketPolicy", "DeleteVectorBucketPolicy", "TagResource", "UntagResource", "ListTagsForResource", "PutVectors", "GetVectors", "ListVectors", "DeleteVectors", "QueryVectors": + default: + writeJSONError(w, http.StatusBadRequest, "Unsupported vector operation") + return + } + switch req.Operation { + case "QueryVectors", "GetVectors", "ListVectors", "ListTagsForResource": + default: + if role := RoleFromContext(r.Context()); role != "admin" && (role != "" || UsernameFromContext(r.Context()) != "") { + writeJSONError(w, http.StatusForbidden, "This operation requires admin access") + return + } + } + var resp json.RawMessage + if err := s.executeS3VectorsOperation(r.Context(), req.Operation, req.Request, &resp); err != nil { + writeS3VectorsError(w, err) + return + } + if len(resp) == 0 { + resp = json.RawMessage(`{}`) + } + writeJSON(w, http.StatusOK, resp) +} diff --git a/weed/admin/dash/s3vectors_management_test.go b/weed/admin/dash/s3vectors_management_test.go index bd0c36a65d..7c0df25df4 100644 --- a/weed/admin/dash/s3vectors_management_test.go +++ b/weed/admin/dash/s3vectors_management_test.go @@ -22,7 +22,9 @@ func newS3VectorsTestServer(t *testing.T) (*AdminServer, *s3tables.ManagerClient server := s3vectors.NewServer(client, nil, nil) server.SetRegion(s3tables.DefaultRegion) server.SetAccountID(s3tables.DefaultAccountID) - return &AdminServer{s3VectorsServer: server}, client + gateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(`{"sseS3":true}`)) })) + t.Cleanup(gateway.Close) + return &AdminServer{s3VectorsServer: server, s3PublicEndpoint: gateway.URL}, client } // createS3VectorsIndexRecord writes the catalog record CreateIndex leaves @@ -172,3 +174,79 @@ func TestS3VectorsBucketsStopAtTheListLimit(t *testing.T) { t.Fatalf("listed %d buckets (truncated %v), want the limit %d and truncated", data.TotalBuckets, data.Truncated, s3TablesAdminListLimit) } } + +func TestS3VectorsManagementOperations(t *testing.T) { + s, _ := newS3VectorsTestServer(t) + w := s3VectorsRequest(t, s.CreateS3VectorsBucket, http.MethodPost, "/api/s3vectors/buckets", `{"name":"managed","encryptionConfiguration":{"sseType":"AES256"},"tags":{"env":"test"}}`) + if w.Code != http.StatusCreated { + t.Fatal(w.Code, w.Body) + } + summary, err := s.s3VectorsBucketSummary(t.Context(), "managed") + if err != nil || summary.Encryption != "AES256" { + t.Fatal(summary, err) + } + arn := summary.ARN + for _, body := range []string{ + `{"operation":"TagResource","request":{"resourceArn":"` + arn + `","tags":{"team":"search"}}}`, + `{"operation":"ListTagsForResource","request":{"resourceArn":"` + arn + `"}}`, + `{"operation":"UntagResource","request":{"resourceArn":"` + arn + `","tagKeys":["team"]}}`, + `{"operation":"PutVectorBucketPolicy","request":{"vectorBucketName":"managed","policy":"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":\"*\",\"Action\":\"s3vectors:QueryVectors\",\"Resource\":\"*\"}]}"}}`, + `{"operation":"DeleteVectorBucketPolicy","request":{"vectorBucketName":"managed"}}`, + } { + w := s3VectorsRequest(t, s.ExecuteS3VectorsOperationAPI, http.MethodPost, "/api/s3vectors/operations", body) + if w.Code != http.StatusOK { + t.Fatalf("%s: %d %s", body, w.Code, w.Body) + } + } + for _, body := range []string{`{"operation":"Unknown","request":{}}`, `{bad json`} { + if w := s3VectorsRequest(t, s.ExecuteS3VectorsOperationAPI, http.MethodPost, "/api/s3vectors/operations", body); w.Code != http.StatusBadRequest { + t.Fatal(w.Code, w.Body) + } + } + request := httptest.NewRequest(http.MethodPost, "/api/s3vectors/operations", strings.NewReader(`{}`)) + request = request.WithContext(WithAuthContext(request.Context(), "admin", "admin", "required-token")) + recorder := httptest.NewRecorder() + s.ExecuteS3VectorsOperationAPI(recorder, request) + if recorder.Code != http.StatusForbidden { + t.Fatal("missing CSRF token accepted", recorder.Code) + } +} + +func TestS3VectorsEncryptedCreationRequiresKey(t *testing.T) { + s, _ := newS3VectorsTestServer(t) + s.s3VectorsServer.SetEncryptionChecker(func() bool { return false }) + w := s3VectorsRequest(t, s.CreateS3VectorsBucket, http.MethodPost, "/api/s3vectors/buckets", `{"name":"sealed","encryptionConfiguration":{"sseType":"AES256"}}`) + if w.Code != http.StatusBadRequest { + t.Fatalf("encrypted bucket without key: %d %s", w.Code, w.Body) + } + data, err := s.GetS3VectorsBucketsData(t.Context()) + if err != nil || len(data.Buckets) != 0 { + t.Fatalf("unusable bucket persisted: %+v %v", data, err) + } +} + +func TestS3VectorsReadOnlyOperations(t *testing.T) { + s, _ := newS3VectorsTestServer(t) + s3VectorsRequest(t, s.CreateS3VectorsBucket, http.MethodPost, "/api/s3vectors/buckets", `{"name":"readable"}`) + summary, err := s.s3VectorsBucketSummary(t.Context(), "readable") + if err != nil { + t.Fatal(err) + } + for _, role := range []string{"readonly", ""} { + for _, operation := range []string{"ListTagsForResource", "TagResource", "CreateIndex", "DeleteVectors"} { + body := fmt.Sprintf(`{"operation":%q,"request":{"resourceArn":%q}}`, operation, summary.ARN) + request := httptest.NewRequest(http.MethodPost, "/api/s3vectors/operations", strings.NewReader(body)) + request = request.WithContext(WithAuthContext(request.Context(), "viewer", role, "token")) + request.Header.Set("X-CSRF-Token", "token") + w := httptest.NewRecorder() + s.ExecuteS3VectorsOperationAPI(w, request) + want := http.StatusForbidden + if operation == "ListTagsForResource" { + want = http.StatusOK + } + if w.Code != want { + t.Fatalf("%s: %d %s", operation, w.Code, w.Body) + } + } + } +} diff --git a/weed/admin/dash/vector_encryption.go b/weed/admin/dash/vector_encryption.go new file mode 100644 index 0000000000..6ec3d0de2b --- /dev/null +++ b/weed/admin/dash/vector_encryption.go @@ -0,0 +1,60 @@ +package dash + +import ( + "context" + "encoding/json" + "io" + "net/http" + "strings" + "time" + + "github.com/seaweedfs/seaweedfs/weed/cluster" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" +) + +func (s *AdminServer) vectorEncryptionConfigured(ctx context.Context) bool { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + endpoints := []string{} + if s.s3PublicEndpoint != "" { + endpoints = append(endpoints, s.s3PublicEndpoint) + } else if s.masterClient != nil { + err := s.masterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error { + response, err := client.ListClusterNodes(ctx, s.listClusterNodesRequest(cluster.S3Type)) + if err != nil { + return err + } + for _, node := range response.ClusterNodes { + endpoints = append(endpoints, "http://"+pb.ServerAddress(node.Address).ToHttpAddress()) + } + return nil + }) + if err != nil { + return false + } + } + if len(endpoints) == 0 { + return false + } + for _, endpoint := range endpoints { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(endpoint, "/")+"/status?features", nil) + if err != nil { + return false + } + response, err := util_http.GetGlobalHttpClient().Do(request) + if err != nil { + return false + } + var features struct { + SSES3 bool `json:"sseS3"` + } + err = json.NewDecoder(io.LimitReader(response.Body, 4096)).Decode(&features) + _ = response.Body.Close() + if response.StatusCode != http.StatusOK || err != nil || !features.SSES3 { + return false + } + } + return true +} diff --git a/weed/admin/dash/vector_encryption_test.go b/weed/admin/dash/vector_encryption_test.go new file mode 100644 index 0000000000..7c46225878 --- /dev/null +++ b/weed/admin/dash/vector_encryption_test.go @@ -0,0 +1,26 @@ +package dash + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestVectorEncryptionChecksRemoteGateway(t *testing.T) { + for _, test := range []struct { + body string + want bool + }{{`{"sseS3":true}`, true}, {`{"sseS3":false}`, false}, {``, false}} { + gateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/status" || !r.URL.Query().Has("features") { + t.Errorf("wrong probe: %s", r.URL) + } + _, _ = w.Write([]byte(test.body)) + })) + s := &AdminServer{s3PublicEndpoint: gateway.URL} + if got := s.vectorEncryptionConfigured(t.Context()); got != test.want { + t.Errorf("body %q: %v", test.body, got) + } + gateway.Close() + } +} diff --git a/weed/admin/dash/vector_status.go b/weed/admin/dash/vector_status.go new file mode 100644 index 0000000000..549e53a598 --- /dev/null +++ b/weed/admin/dash/vector_status.go @@ -0,0 +1,107 @@ +package dash + +import ( + "context" + "sort" + "time" + + adminplugin "github.com/seaweedfs/seaweedfs/weed/admin/plugin" + "github.com/seaweedfs/seaweedfs/weed/cluster" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +type VectorNode struct { + Address string `json:"address"` + Version string `json:"version"` + Status string `json:"status"` +} + +type VectorOperationsData struct { + Engines []VectorNode `json:"engines"` + DiscoveryError string `json:"discovery_error,omitempty"` + MaintenanceWorkers int `json:"maintenance_workers"` + MaintenanceJobs []adminplugin.TrackedJob `json:"maintenance_jobs"` +} + +func (s *AdminServer) getVectorNodesStatus(ctx context.Context) VectorOperationsData { + data := VectorOperationsData{Engines: []VectorNode{}} + if s.masterClient == nil { + data.DiscoveryError = "Master discovery unavailable" + return data + } + var discoveryErr error + for _, master := range s.masterClient.ListCachedMasters() { + discoveryCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + conn, err := pb.GrpcDial(discoveryCtx, master.ToGrpcAddress(), false, s.grpcDialOption) + if err == nil { + response, callErr := master_pb.NewSeaweedClient(conn).ListClusterNodes(discoveryCtx, s.listClusterNodesRequest(cluster.VectorType)) + _ = conn.Close() + err = callErr + if err == nil { + for _, node := range response.ClusterNodes { + data.Engines = append(data.Engines, VectorNode{Address: node.Address, Version: node.Version, Status: "Registered"}) + } + } + } + cancel() + if err != nil { + discoveryErr = err + } + if len(data.Engines) > 0 { + break + } + } + if len(data.Engines) == 0 && discoveryErr != nil { + data.DiscoveryError = discoveryErr.Error() + return data + } + var checks errgroup.Group + checks.SetLimit(8) + for i := range data.Engines { + checks.Go(func() error { + data.Engines[i].Status = vectorNodeStatus(ctx, data.Engines[i].Address, s.grpcDialOption) + return nil + }) + } + _ = checks.Wait() + sort.Slice(data.Engines, func(i, j int) bool { return data.Engines[i].Address < data.Engines[j].Address }) + return data +} + +func vectorNodeStatus(ctx context.Context, address string, dialOption grpc.DialOption) string { + ctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + conn, err := pb.GrpcDial(ctx, address, false, dialOption) + if err != nil { + return "Unreachable" + } + defer conn.Close() + response, err := grpc_health_v1.NewHealthClient(conn).Check(ctx, &grpc_health_v1.HealthCheckRequest{Service: "vector_pb.VectorService"}) + if status.Code(err) == codes.Unimplemented || status.Code(err) == codes.NotFound { + return "Registered (health unavailable)" + } + if err != nil { + return "Unreachable" + } + if response.Status == grpc_health_v1.HealthCheckResponse_SERVING { + return "Ready" + } + return "Not ready" +} + +func (s *AdminServer) GetVectorOperationsData(ctx context.Context) VectorOperationsData { + data := s.getVectorNodesStatus(ctx) + for _, worker := range s.GetPluginWorkers() { + if capability := worker.Capabilities["lance_vector_index"]; capability != nil && capability.CanExecute { + data.MaintenanceWorkers++ + } + } + data.MaintenanceJobs = s.ListPluginJobs("lance_vector_index", "", 5) + return data +} diff --git a/weed/admin/dash/vector_status_test.go b/weed/admin/dash/vector_status_test.go new file mode 100644 index 0000000000..f37c4c378b --- /dev/null +++ b/weed/admin/dash/vector_status_test.go @@ -0,0 +1,49 @@ +package dash + +import ( + "context" + "net" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health" + "google.golang.org/grpc/health/grpc_health_v1" +) + +func TestVectorNodeStatus(t *testing.T) { + for _, want := range []string{"Ready", "Not ready", "Registered (health unavailable)", "Unknown service"} { + t.Run(want, func(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + server := grpc.NewServer() + if want != "Registered (health unavailable)" { + checker := health.NewServer() + state := grpc_health_v1.HealthCheckResponse_NOT_SERVING + if want == "Ready" { + state = grpc_health_v1.HealthCheckResponse_SERVING + } + if want != "Unknown service" { + checker.SetServingStatus("vector_pb.VectorService", state) + } + grpc_health_v1.RegisterHealthServer(server, checker) + } + go server.Serve(listener) + t.Cleanup(server.Stop) + expected := want + if want == "Unknown service" { + expected = "Registered (health unavailable)" + } + if got := vectorNodeStatus(t.Context(), listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())); got != expected { + t.Fatalf("got %s want %s", got, want) + } + }) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if got := vectorNodeStatus(ctx, "127.0.0.1:1", grpc.WithTransportCredentials(insecure.NewCredentials())); got != "Unreachable" { + t.Fatal(got) + } +} diff --git a/weed/admin/handlers/admin_handlers.go b/weed/admin/handlers/admin_handlers.go index 80018e1d76..3ff3681e42 100644 --- a/weed/admin/handlers/admin_handlers.go +++ b/weed/admin/handlers/admin_handlers.go @@ -256,6 +256,7 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) { s3TablesApi.Handle("/tags", wrapWrite(h.adminServer.UntagS3TablesResource)).Methods(http.MethodDelete) s3VectorsApi := api.PathPrefix("/s3vectors").Subrouter() + s3VectorsApi.HandleFunc("/operations", h.adminServer.ExecuteS3VectorsOperationAPI).Methods(http.MethodPost) s3VectorsApi.HandleFunc("/buckets", h.adminServer.ListS3VectorsBucketsAPI).Methods(http.MethodGet) s3VectorsApi.Handle("/buckets", wrapWrite(h.adminServer.CreateS3VectorsBucket)).Methods(http.MethodPost) s3VectorsApi.Handle("/buckets", wrapWrite(h.adminServer.DeleteS3VectorsBucket)).Methods(http.MethodDelete) diff --git a/weed/admin/view/app/policy_action_suggestions.go b/weed/admin/view/app/policy_action_suggestions.go index 2bb11ef170..990312d098 100644 --- a/weed/admin/view/app/policy_action_suggestions.go +++ b/weed/admin/view/app/policy_action_suggestions.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" ) // PolicyActionSuggestions feeds the used by the structured policy @@ -26,7 +27,7 @@ var S3TablesPolicyActionSuggestions = func() []string { }() func buildPolicyActionSuggestions() []string { - return []string{ + suggestions := []string{ // s3: actions, sourced from the constants used elsewhere for policy // evaluation (weed/s3api/s3_constants/s3_action_strings.go) so the // suggestion list can't drift from the strings the engine actually @@ -120,4 +121,10 @@ func buildPolicyActionSuggestions() []string { s3_constants.S3TABLES_ACTION_LIST_TAGS_FOR_RESOURCE, s3_constants.S3TABLES_ACTION_UNTAG_RESOURCE, } + suggestions = append(suggestions, "s3vectors:*") + for _, operation := range s3vectors.Operations() { + suggestions = append(suggestions, "s3vectors:"+operation) + } + return suggestions + } diff --git a/weed/admin/view/app/policy_action_suggestions_test.go b/weed/admin/view/app/policy_action_suggestions_test.go index d45cee4204..ab01b88df4 100644 --- a/weed/admin/view/app/policy_action_suggestions_test.go +++ b/weed/admin/view/app/policy_action_suggestions_test.go @@ -23,7 +23,7 @@ func TestPolicyActionSuggestions_NoDuplicates(t *testing.T) { func TestPolicyActionSuggestions_AllPrefixed(t *testing.T) { for _, action := range PolicyActionSuggestions { - if !strings.HasPrefix(action, "s3:") && !strings.HasPrefix(action, "s3tables:") { + if !strings.HasPrefix(action, "s3:") && !strings.HasPrefix(action, "s3tables:") && !strings.HasPrefix(action, "s3vectors:") { t.Errorf("action suggestion %q is not prefixed with a known service", action) } } @@ -36,3 +36,17 @@ func TestPolicyActionSuggestions_NoEmptyStrings(t *testing.T) { } } } + +func TestVectorPolicySuggestions(t *testing.T) { + for _, want := range []string{"s3vectors:*", "s3vectors:QueryVectors", "s3vectors:CreateIndex", "s3vectors:TagResource"} { + found := false + for _, value := range PolicyActionSuggestions { + if value == want { + found = true + } + } + if !found { + t.Errorf("missing %s", want) + } + } +} diff --git a/weed/admin/view/app/s3vectors_bucket_details.templ b/weed/admin/view/app/s3vectors_bucket_details.templ index 8a6bf1c21d..204ab01ff1 100644 --- a/weed/admin/view/app/s3vectors_bucket_details.templ +++ b/weed/admin/view/app/s3vectors_bucket_details.templ @@ -23,6 +23,8 @@ templ S3VectorsBucketDetails(data dash.S3VectorsBucketData) { + @VectorStatus(data.Operations) + @VectorTools(data.Bucket.Name,data.Bucket.ARN,data.Policy)
@@ -92,7 +94,7 @@ templ S3VectorsBucketDetails(data dash.S3VectorsBucketData) { if len(data.Indexes) == 0 { - No indexes. Create one with an S3 Vectors client, e.g. aws s3vectors create-index. + No indexes. Create one with the form above. } diff --git a/weed/admin/view/app/s3vectors_bucket_details_templ.go b/weed/admin/view/app/s3vectors_bucket_details_templ.go index c74e739b65..880f0a88ed 100644 --- a/weed/admin/view/app/s3vectors_bucket_details_templ.go +++ b/weed/admin/view/app/s3vectors_bucket_details_templ.go @@ -74,163 +74,162 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "
ARN
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = VectorStatus(data.Operations).Render(ctx, templ_7745c5c3_Buffer) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = VectorTools(data.Bucket.Name, data.Bucket.ARN, data.Policy).Render(ctx, templ_7745c5c3_Buffer) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "
ARN
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var5 string templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.ARN) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 30, Col: 49} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 32, Col: 49} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "
Created
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "
Created
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var6 string templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.CreatedAt.Format("2006-01-02 15:04:05")) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 32, Col: 79} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 34, Col: 79} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "
Encryption
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "
Encryption
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.Bucket.Encryption != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var7 string templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Encryption) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 36, Col: 61} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 38, Col: 61} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "None") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "None") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var8 string templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d indexes", data.TotalIndexes)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 47, Col: 84} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 49, Col: 84} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.Truncated { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var9 string templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalIndexes)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 49, Col: 138} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 51, Col: 138} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "
NameData TypeDimensionDistance MetricNon-filterable Metadata KeysCreatedActions
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } for _, index := range data.Indexes { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "\" title=\"Delete\">") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } if len(data.Indexes) == 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "
NameData TypeDimensionDistance MetricNon-filterable Metadata KeysCreatedActions
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var10 string templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(index.IndexName) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 71, Col: 26} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 73, Col: 26} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var11 string templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(index.IndexArn) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 72, Col: 55} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 74, Col: 55} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var12 string templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(index.DataType) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 74, Col: 28} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 76, Col: 28} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var13 string - templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", index.Dimension)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 75, Col: 48} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var14 string - templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(index.DistanceMetric) + var templ_7745c5c3_Var13 string + templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", index.Dimension)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 76, Col: 34} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 77, Col: 48} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -238,109 +237,122 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } + var templ_7745c5c3_Var14 string + templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(index.DistanceMetric) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 78, Col: 34} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } if index.MetadataConfiguration != nil && len(index.MetadataConfiguration.NonFilterableMetadataKeys) > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var15 string templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(strings.Join(index.MetadataConfiguration.NonFilterableMetadataKeys, ", ")) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 79, Col: 91} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 81, Col: 91} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "None") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "None") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var16 string templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(index.CreationTime.Format("2006-01-02 15:04")) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 84, Col: 59} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 86, Col: 59} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "
No indexes. Create one with an S3 Vectors client, e.g. aws s3vectors create-index.
No indexes. Create one with the form above.
Bucket Policy
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "
Bucket Policy
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.Policy != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "
")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "
")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var18 string
 			templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(indentJSON(data.Policy))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 112, Col: 102}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 114, Col: 102}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "
")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "No policy. Set one with PutVectorBucketPolicy.") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "No policy. Set one with PutVectorBucketPolicy.") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "
Delete Index

Delete the index and all its vectors?

Delete Index

Delete the index and all its vectors?

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "\">") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/admin/view/app/s3vectors_buckets.templ b/weed/admin/view/app/s3vectors_buckets.templ index 626212dd01..bb42466a2d 100644 --- a/weed/admin/view/app/s3vectors_buckets.templ +++ b/weed/admin/view/app/s3vectors_buckets.templ @@ -23,8 +23,9 @@ templ S3VectorsBuckets(data dash.S3VectorsBucketsData) {
Vector buckets serve the Amazon S3 Vectors API on the S3 port. Each one is a LANCE table bucket that keeps its indexes in its vector namespace. - Indexes are created and filled with an S3 Vectors client, which needs a running weed-vector engine. + Create indexes and manage vectors from a bucket page or an S3 Vectors client.
+ @VectorStatus(data.Operations)
@@ -104,9 +105,10 @@ templ S3VectorsBuckets(data dash.S3VectorsBucketsData) {
3 to 63 lowercase letters, digits and hyphens, starting and ending with a letter or digit.
-
- The bucket is created without encryption. To encrypt it, create it with an S3 Vectors client and encryptionConfiguration, which the S3 gateway checks against its SSE-S3 key. -
+ + +
SSE-S3 requires the encryption key on the S3 gateway and vector engines.
+
@@ -172,7 +174,7 @@ templ S3VectorsBuckets(data dash.S3VectorsBucketsData) { const response = await fetch(s3tBasePath('/api/s3vectors/buckets'), { method: 'POST', headers: s3tWriteHeaders({ 'Content-Type': 'application/json' }), - body: JSON.stringify({ name: name }) + body: JSON.stringify({ name, encryptionConfiguration: document.getElementById('s3vectorsEncryption').value ? {sseType:'AES256'} : undefined }) }); const data = await response.json(); if (!response.ok) { diff --git a/weed/admin/view/app/s3vectors_buckets_templ.go b/weed/admin/view/app/s3vectors_buckets_templ.go index 8b486b043a..7212b9dd7d 100644 --- a/weed/admin/view/app/s3vectors_buckets_templ.go +++ b/weed/admin/view/app/s3vectors_buckets_templ.go @@ -33,165 +33,173 @@ func S3VectorsBuckets(data dash.S3VectorsBucketsData) templ.Component { } ctx = templ.InitializeContext(ctx) ctx = templ.ClearChildren(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

Vector Buckets

Vector buckets serve the Amazon S3 Vectors API on the S3 port. Each one is a LANCE table bucket that keeps its indexes in its vector namespace. Indexes are created and filled with an S3 Vectors client, which needs a running weed-vector engine.
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

Vector Buckets

Vector buckets serve the Amazon S3 Vectors API on the S3 port. Each one is a LANCE table bucket that keeps its indexes in its vector namespace. Create indexes and manage vectors from a bucket page or an S3 Vectors client.
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = VectorStatus(data.Operations).Render(ctx, templ_7745c5c3_Buffer) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var2 string templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d vector buckets", data.TotalBuckets)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 31, Col: 104} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 32, Col: 104} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.Truncated { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var3 string templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalBuckets)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 33, Col: 138} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 34, Col: 138} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "
NameARNEncryptionCreatedActions
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } for _, bucket := range data.Buckets { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "\" title=\"Delete\">") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } if len(data.Buckets) == 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "
NameARNEncryptionCreatedActions
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "\">") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var5 string templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 53, Col: 110} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 54, Col: 110} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var6 string templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 55, Col: 49} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 56, Col: 49} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if bucket.Encryption != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var7 string templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Encryption) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 58, Col: 60} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 59, Col: 60} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "None") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "None") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var8 string templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04")) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 63, Col: 57} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 64, Col: 57} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "
No vector buckets found
No vector buckets found
Create Vector Bucket
3 to 63 lowercase letters, digits and hyphens, starting and ending with a letter or digit.
The bucket is created without encryption. To encrypt it, create it with an S3 Vectors client and encryptionConfiguration, which the S3 gateway checks against its SSE-S3 key.
Delete Vector Bucket

Delete the vector bucket ? A bucket that still has indexes is not deleted.

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "
Create Vector Bucket
3 to 63 lowercase letters, digits and hyphens, starting and ending with a letter or digit.
SSE-S3 requires the encryption key on the S3 gateway and vector engines.
Delete Vector Bucket

Delete the vector bucket ? A bucket that still has indexes is not deleted.

") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/admin/view/app/s3vectors_buckets_test.go b/weed/admin/view/app/s3vectors_buckets_test.go index bae344eb2d..ee97245d23 100644 --- a/weed/admin/view/app/s3vectors_buckets_test.go +++ b/weed/admin/view/app/s3vectors_buckets_test.go @@ -78,3 +78,21 @@ func TestS3TablesBucketsLinksVectorBuckets(t *testing.T) { require.Contains(t, page, `href="/object-store/vector-buckets/vectors"`) require.NotContains(t, page, `href="/object-store/vector-buckets/warehouse"`) } + +func TestVectorOperationalStates(t *testing.T) { + for _, data := range []dash.VectorOperationsData{ + {}, {DiscoveryError: "master unavailable"}, + {Engines: []dash.VectorNode{{Address: "engine:8090", Status: "Ready"}}, MaintenanceWorkers: 1}, + } { + var out bytes.Buffer + require.NoError(t, VectorStatus(data).Render(t.Context(), &out)) + if data.DiscoveryError != "" { + require.Contains(t, out.String(), "Engine discovery failed") + } else if len(data.Engines) == 0 { + require.Contains(t, out.String(), "No vector engine is registered") + } else { + require.Contains(t, out.String(), "engine:8090") + require.Contains(t, out.String(), "Ready") + } + } +} diff --git a/weed/admin/view/app/vector_status.templ b/weed/admin/view/app/vector_status.templ new file mode 100644 index 0000000000..9318e55974 --- /dev/null +++ b/weed/admin/view/app/vector_status.templ @@ -0,0 +1,38 @@ +package app + +import ( + "fmt" + "github.com/seaweedfs/seaweedfs/weed/admin/dash" +) + +templ VectorStatus(data dash.VectorOperationsData) { +
+
Vector Engines and Maintenance
+
+ if data.DiscoveryError != "" { +
Engine discovery failed: { data.DiscoveryError }
+ } else if len(data.Engines)==0 { +
No vector engine is registered. Index creation, ingestion and search need a running engine.
+ } else { + + for _,engine:=range data.Engines { + + } +
EngineVersionStatus
{ engine.Address }{ engine.Version }{ engine.Status }
+ } + if data.MaintenanceWorkers == 0 { +

No vector index maintenance worker is connected. Searches still work; enable a Lance worker to build and refresh ANN indexes as data grows.

+ } else { +

{ fmt.Sprintf("%d vector maintenance workers connected",data.MaintenanceWorkers) }

+ } + Maintenance jobs + if len(data.MaintenanceJobs)>0 { + + for _,job:=range data.MaintenanceJobs { + + } +
Recent jobStateResult
{ job.JobID }{ job.State }{ job.ResultSummary } { job.ErrorMessage }
+ } +
+
+} diff --git a/weed/admin/view/app/vector_status_templ.go b/weed/admin/view/app/vector_status_templ.go new file mode 100644 index 0000000000..ba03e203b4 --- /dev/null +++ b/weed/admin/view/app/vector_status_templ.go @@ -0,0 +1,233 @@ +// Code generated by templ - DO NOT EDIT. + +// templ: version: v0.3.1100 + +package app + +//lint:file-ignore SA4006 This context is only used if a nested component is present. + +import "github.com/a-h/templ" +import templruntime "github.com/a-h/templ/runtime" + +import ( + "fmt" + "github.com/seaweedfs/seaweedfs/weed/admin/dash" +) + +func VectorStatus(data dash.VectorOperationsData) templ.Component { + return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { + templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context + if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { + return templ_7745c5c3_CtxErr + } + templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) + if !templ_7745c5c3_IsBuffer { + defer func() { + templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) + if templ_7745c5c3_Err == nil { + templ_7745c5c3_Err = templ_7745c5c3_BufErr + } + }() + } + ctx = templ.InitializeContext(ctx) + ctx = templ.ClearChildren(ctx) + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "
Vector Engines and Maintenance
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if data.DiscoveryError != "" { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "
Engine discovery failed: ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var2 string + templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(data.DiscoveryError) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 13, Col: 83} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else if len(data.Engines) == 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "
No vector engine is registered. Index creation, ingestion and search need a running engine.
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + for _, engine := range data.Engines { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "
EngineVersionStatus
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var3 string + templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(engine.Address) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 19, Col: 30} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var4 string + templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(engine.Version) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 19, Col: 57} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var5 string + templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(engine.Status) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 19, Col: 83} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + if data.MaintenanceWorkers == 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "

No vector index maintenance worker is connected. Searches still work; enable a Lance worker to build and refresh ANN indexes as data grows.

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var6 string + templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d vector maintenance workers connected", data.MaintenanceWorkers)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 26, Col: 87} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "Maintenance jobs ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if len(data.MaintenanceJobs) > 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + for _, job := range data.MaintenanceJobs { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "
Recent jobStateResult
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var8 string + templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(job.JobID) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 32, Col: 25} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var9 string + templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(job.State) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 32, Col: 47} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var10 string + templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(job.ResultSummary) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 32, Col: 77} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, " ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var11 string + templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(job.ErrorMessage) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 32, Col: 98} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + return nil + }) +} + +var _ = templruntime.GeneratedTemplate diff --git a/weed/admin/view/app/vector_tools.templ b/weed/admin/view/app/vector_tools.templ new file mode 100644 index 0000000000..05a4ed7911 --- /dev/null +++ b/weed/admin/view/app/vector_tools.templ @@ -0,0 +1,85 @@ +package app + + +templ VectorTools(bucket, arn, policy string) { +
+
Manage Indexes, Policies and Vectors
+
+
+
+
+
+
+
+
+ + + +
+
+ + + + +
+ 

+  

+ 

+ 
+}
diff --git a/weed/admin/view/app/vector_tools_templ.go b/weed/admin/view/app/vector_tools_templ.go
new file mode 100644
index 0000000000..77b7db414e
--- /dev/null
+++ b/weed/admin/view/app/vector_tools_templ.go
@@ -0,0 +1,76 @@
+// Code generated by templ - DO NOT EDIT.
+
+// templ: version: v0.3.1100
+
+package app
+
+//lint:file-ignore SA4006 This context is only used if a nested component is present.
+
+import "github.com/a-h/templ"
+import templruntime "github.com/a-h/templ/runtime"
+
+func VectorTools(bucket, arn, policy string) templ.Component {
+	return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
+		templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
+		if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
+			return templ_7745c5c3_CtxErr
+		}
+		templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
+		if !templ_7745c5c3_IsBuffer {
+			defer func() {
+				templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
+				if templ_7745c5c3_Err == nil {
+					templ_7745c5c3_Err = templ_7745c5c3_BufErr
+				}
+			}()
+		}
+		ctx = templ.InitializeContext(ctx)
+		ctx = templ.ClearChildren(ctx)
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "
Manage Indexes, Policies and Vectors
  
   
")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		return nil
+	})
+}
+
+var _ = templruntime.GeneratedTemplate
diff --git a/weed/command/admin.go b/weed/command/admin.go
index b2e6e7854e..ba44c692a9 100644
--- a/weed/command/admin.go
+++ b/weed/command/admin.go
@@ -516,14 +516,6 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,
 		return fmt.Errorf("apply admin.toml to plugin config: %w", err)
 	}
 
-	// Show discovered filers
-	filers := adminServer.GetAllFilers()
-	if len(filers) > 0 {
-		glog.Infof("Discovered filers: %s", strings.Join(filers, ", "))
-	} else {
-		glog.Infof("No filers discovered from masters")
-	}
-
 	// Start worker gRPC server for worker connections. The worker gRPC binds
 	// to its own address (workerGrpcBindIp) which, unlike the HTTP ip, does
 	// not auto-upgrade to 0.0.0.0 based on adminPassword, since the worker
diff --git a/weed/command/mini.go b/weed/command/mini.go
index 5f906cdd6c..ffbdf4f5da 100644
--- a/weed/command/mini.go
+++ b/weed/command/mini.go
@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"maps"
 	"math/bits"
 	"net"
 	"net/http"
@@ -415,6 +416,11 @@ var (
 	miniS3AutoCreateBucket          = cmdMini.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
 	miniBucket                      = cmdMini.Flag.String("bucket", "", "comma-separated S3 bucket names to create on startup if they do not already exist; leave empty to skip. Falls back to S3_BUCKET env var.")
 	miniTableBucket                 = cmdMini.Flag.String("tableBucket", "", "comma-separated S3 Tables buckets to create on startup if they do not already exist, each name[:FORMAT] with FORMAT one of ICEBERG (default) or LANCE, e.g. warehouse,vectors:LANCE; leave empty to skip. Falls back to S3_TABLE_BUCKET env var.")
+	miniVector                      = cmdMini.Flag.Bool("vector", false, "start the weed-vector engine for vector indexes and search")
+	miniVectorBinary                = cmdMini.Flag.String("vector.binary", "weed-vector", "path to the vector engine binary")
+	miniVectorPort                  = cmdMini.Flag.Int("vector.port", 8090, "vector engine gRPC port")
+	miniVectorMetricsPort           = cmdMini.Flag.Int("vector.metricsPort", 0, "optional vector HTTP health and metrics port")
+	miniVectorBucket                = cmdMini.Flag.String("vectorBucket", "", "comma-separated vector buckets to create; falls back to S3_VECTOR_BUCKET")
 )
 
 // getBindIp determines the bind IP address based on miniIp and miniBindIp flags
@@ -874,9 +880,9 @@ func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reserv
 	// Skip if this port is reserved for gRPC calculation
 	if reservedPorts[original] {
 		if isExplicitPort {
-			return fmt.Errorf("port %d for %s (specified by flag %s) is reserved for gRPC calculation and cannot be used", original, serviceName, flagName)
+			return fmt.Errorf("port %d for %s (specified by flag %s) is reserved by another mini service and cannot be used", original, serviceName, flagName)
 		}
-		glog.Warningf("Port %d for %s is reserved for gRPC calculation, finding alternative...", original, serviceName)
+		glog.Warningf("Port %d for %s is reserved by another mini service, finding alternative...", original, serviceName)
 		newPort := findAvailablePortOnIP(ip, original+1, 100, reservedPorts, listenOnLocalhost)
 		if newPort == 0 {
 			glog.Errorf("Could not find available port for %s starting from %d, will use original %d and fail on binding", serviceName, original+1, original)
@@ -950,22 +956,41 @@ func ensureAllPortsAvailableOnIP(bindIp string, listenOnLocalhost bool) error {
 			}{miniS3Options.portLance, "Lance", "s3.port.lance", nil, listenOnLocalhost})
 		}
 	}
-	// WebDAV and the Admin UI bind only -ip.bind; they never open a loopback
-	// companion listener, so ports held elsewhere cannot collide with them.
+	if *miniVector {
+		portConfigs = append(portConfigs, struct {
+			port      *int
+			name      string
+			flagName  string
+			grpcPtr   *int
+			localhost bool
+		}{miniVectorPort, "Vector", "vector.port", nil, false})
+		if *miniVectorMetricsPort > 0 {
+			portConfigs = append(portConfigs, struct {
+				port      *int
+				name      string
+				flagName  string
+				grpcPtr   *int
+				localhost bool
+			}{miniVectorMetricsPort, "Vector metrics", "vector.metricsPort", nil, false})
+		}
+	}
+
+	if *miniEnableWebDAV {
+		portConfigs = append(portConfigs, struct {
+			port      *int
+			name      string
+			flagName  string
+			grpcPtr   *int
+			localhost bool
+		}{miniWebDavOptions.port, "WebDAV", "webdav.port", nil, false})
+	}
 	portConfigs = append(portConfigs, struct {
 		port      *int
 		name      string
 		flagName  string
 		grpcPtr   *int
 		localhost bool
-	}{miniWebDavOptions.port, "WebDAV", "webdav.port", nil, false},
-		struct {
-			port      *int
-			name      string
-			flagName  string
-			grpcPtr   *int
-			localhost bool
-		}{miniAdminOptions.port, "Admin", "admin.port", miniAdminOptions.grpcPort, false})
+	}{miniAdminOptions.port, "Admin", "admin.port", miniAdminOptions.grpcPort, false})
 
 	// First, reserve all gRPC ports that will be calculated to prevent HTTP port allocation from using them
 	// This prevents collisions like: HTTP port moves to X, then gRPC port is calculated as Y where Y == X
@@ -978,6 +1003,16 @@ func ensureAllPortsAvailableOnIP(bindIp string, listenOnLocalhost bool) error {
 		}
 	}
 
+	chosenPorts := make(map[int]string)
+	for _, config := range portConfigs {
+		if *config.port > 0 && explicitFlags[config.flagName] {
+			if previous, exists := chosenPorts[*config.port]; exists {
+				return fmt.Errorf("port %d is shared by explicit flags %s and %s", *config.port, previous, config.flagName)
+			}
+			chosenPorts[*config.port] = config.flagName
+		}
+	}
+
 	applyMiniAdminCredentialFallback(&miniAdminOptions)
 	adminBindIP := miniAdminEffectiveBindIP(bindIp)
 
@@ -991,9 +1026,19 @@ func ensureAllPortsAvailableOnIP(bindIp string, listenOnLocalhost bool) error {
 			checkIP = adminBindIP
 		}
 		original := *config.port
-		if err := ensurePortAvailableOnIP(config.port, config.name, checkIP, reservedPorts, config.flagName, config.localhost); err != nil {
+		unavailablePorts := maps.Clone(reservedPorts)
+		for port, flag := range chosenPorts {
+			if flag != config.flagName {
+				unavailablePorts[port] = true
+			}
+		}
+
+		if err := ensurePortAvailableOnIP(config.port, config.name, checkIP, unavailablePorts, config.flagName, config.localhost); err != nil {
 			return err
 		}
+		if *config.port > 0 {
+			chosenPorts[*config.port] = config.flagName
+		}
 		// If port was changed, update the reserved gRPC ports mapping
 		if *config.port != original && config.grpcPtr != nil && *config.grpcPtr == 0 {
 			delete(reservedPorts, original+GrpcPortOffset)
@@ -1105,6 +1150,13 @@ func initializeGrpcPortsOnIP(bindIp string, listenOnLocalhost bool) map[int]bool
 		}
 	}
 
+	if *miniVector {
+		allocatedPorts[*miniVectorPort] = true
+		if *miniVectorMetricsPort > 0 {
+			allocatedPorts[*miniVectorMetricsPort] = true
+		}
+	}
+
 	grpcConfigs := []struct {
 		httpPort  *int
 		grpcPort  *int
@@ -1641,6 +1693,32 @@ func runMini(cmd *Command, args []string) bool {
 		os.Unsetenv("S3_TABLE_BUCKET")
 	}
 
+	vectorBuckets := *miniVectorBucket
+	if vectorBuckets == "" {
+		vectorBuckets = os.Getenv("S3_VECTOR_BUCKET")
+	}
+	var vectorEntries []tableBucketEntry
+	for _, name := range parseBucketList(vectorBuckets) {
+		vectorEntries = append(vectorEntries, tableBucketEntry{name: name, format: s3tables.FormatLance})
+	}
+	if len(vectorEntries) > 0 && !*miniEnableS3 {
+		glog.Errorf("vector buckets require the S3 gateway")
+		triggerMiniClientsShutdown(5 * time.Second)
+		return false
+	}
+	if ready, err := createMiniTableBuckets(vectorEntries); err != nil || len(ready) != len(vectorEntries) {
+		glog.Errorf("created %d of %d requested vector buckets: %v", len(ready), len(vectorEntries), err)
+		triggerMiniClientsShutdown(5 * time.Second)
+		return false
+	}
+	if *miniVector {
+		if err := startMiniVector(); err != nil {
+			glog.Errorf("start vector engine: %v", err)
+			triggerMiniClientsShutdown(5 * time.Second)
+			return false
+		}
+	}
+
 	// Print welcome message after all services are running
 	printWelcomeMessage()
 
@@ -2423,6 +2501,14 @@ func ensureMiniTableBuckets(buckets []tableBucketEntry) ([]tableBucketEntry, err
 		return nil, nil
 	}
 
+	return createMiniTableBuckets(servable)
+}
+
+func createMiniTableBuckets(servable []tableBucketEntry) ([]tableBucketEntry, error) {
+	if len(servable) == 0 {
+		return nil, nil
+	}
+
 	filerAddress := pb.NewServerAddress(*miniIp, *miniFilerOptions.port, *miniFilerOptions.portGrpc)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
diff --git a/weed/command/mini_vector.go b/weed/command/mini_vector.go
new file mode 100644
index 0000000000..c2420d82f0
--- /dev/null
+++ b/weed/command/mini_vector.go
@@ -0,0 +1,130 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func startMiniVector() error {
+	binary := *miniVectorBinary
+	if binary == "weed-vector" {
+		if executable, err := os.Executable(); err == nil {
+			candidate := filepath.Join(filepath.Dir(executable), binary)
+			if resolved, err := exec.LookPath(candidate); err == nil {
+				binary = resolved
+			}
+		}
+	}
+	if _, err := exec.LookPath(binary); err != nil {
+		return fmt.Errorf("%s is unavailable: install the vector component or set -vector.binary: %w", binary, err)
+	}
+	address := util.JoinHostPort(*miniIp, *miniVectorPort)
+	bindIP := *miniBindIp
+	if bindIP == "" {
+		bindIP = *miniIp
+	}
+	if net.ParseIP(bindIP) == nil {
+		ctx, cancel := context.WithTimeout(miniClientsCtx(), 5*time.Second)
+		addresses, err := net.DefaultResolver.LookupIPAddr(ctx, bindIP)
+		cancel()
+		if err != nil {
+			return fmt.Errorf("resolve vector bind address %s: %w", bindIP, err)
+		}
+		if len(addresses) == 0 {
+			return fmt.Errorf("vector bind address %s has no IP addresses", bindIP)
+		}
+		bindIP = addresses[0].String()
+	}
+
+	args := []string{"--listen=" + util.JoinHostPort(bindIP, *miniVectorPort), "--advertise=" + address, "--masters=" + string(pb.NewServerAddress(*miniIp, *miniMasterOptions.port, *miniMasterOptions.portGrpc))}
+	if miniFilerOptions.filerGroup != nil && *miniFilerOptions.filerGroup != "" {
+		args = append(args, "--filer-group="+*miniFilerOptions.filerGroup)
+	}
+	if *miniVectorMetricsPort > 0 {
+		args = append(args, fmt.Sprintf("--metrics-port=%d", *miniVectorMetricsPort), "--metrics-ip="+bindIP)
+	}
+	if directory := util.ConfigurationFileDirectory.String(); directory != "" {
+		args = append(args, "--config-dir="+util.ResolvePath(directory))
+	}
+	command := exec.CommandContext(miniClientsCtx(), binary, args...)
+	command.Stdout, command.Stderr = os.Stdout, os.Stderr
+	if err := command.Start(); err != nil {
+		return err
+	}
+	done := trackMiniClient()
+	exited := make(chan error, 1)
+	go func() {
+		defer done()
+		err := command.Wait()
+		exited <- err
+		if err != nil && miniClientsCtx().Err() == nil {
+			glog.Errorf("vector engine exited: %v", err)
+		}
+	}()
+	ctx, cancel := context.WithTimeout(miniClientsCtx(), 30*time.Second)
+	defer cancel()
+	ticker := time.NewTicker(100 * time.Millisecond)
+	defer ticker.Stop()
+	for {
+		select {
+		case err := <-exited:
+			return fmt.Errorf("vector engine exited before readiness: %v", err)
+		case <-ctx.Done():
+			_ = command.Process.Kill()
+			return ctx.Err()
+		case <-ticker.C:
+			conn, err := net.DialTimeout("tcp", miniVectorLocalAddress(bindIP, *miniVectorPort), 100*time.Millisecond)
+			if err == nil {
+				_ = conn.Close()
+				if !miniVectorRegistered(ctx, address) {
+					continue
+				}
+				glog.V(0).Infof("vector engine ready on %s", address)
+				return nil
+			}
+		}
+	}
+}
+
+func miniVectorLocalAddress(bindIP string, port int) string {
+	if ip := net.ParseIP(bindIP); ip != nil && ip.IsUnspecified() {
+		if ip.To4() != nil {
+			bindIP = "127.0.0.1"
+		} else {
+			bindIP = "::1"
+		}
+	}
+	return util.JoinHostPort(bindIP, port)
+}
+
+func miniVectorRegistered(ctx context.Context, address string) bool {
+	ctx, cancel := context.WithTimeout(ctx, time.Second)
+	defer cancel()
+	master := pb.NewServerAddress(*miniIp, *miniMasterOptions.port, *miniMasterOptions.portGrpc)
+	ready := false
+	_ = pb.WithMasterClient(ctx, false, master, security.LoadClientTLS(util.GetViper(), "grpc.client"), false, func(client master_pb.SeaweedClient) error {
+		response, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{ClientType: "vector", FilerGroup: *miniFilerOptions.filerGroup})
+		if err != nil {
+			return err
+		}
+		for _, node := range response.ClusterNodes {
+			if node.Address == address {
+				ready = true
+				break
+			}
+		}
+		return nil
+	})
+	return ready
+}
diff --git a/weed/command/mini_vector_test.go b/weed/command/mini_vector_test.go
new file mode 100644
index 0000000000..4027e49a81
--- /dev/null
+++ b/weed/command/mini_vector_test.go
@@ -0,0 +1,21 @@
+package command
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestMiniVectorRejectsExplicitListenerConflict(t *testing.T) {
+	oldVector, oldPort, oldMetrics, oldFlags := *miniVector, *miniVectorPort, *miniVectorMetricsPort, explicitFlags
+	defer func() {
+		*miniVector, *miniVectorPort, *miniVectorMetricsPort, explicitFlags = oldVector, oldPort, oldMetrics, oldFlags
+	}()
+	*miniVector = true
+	*miniVectorPort = 8090
+	*miniVectorMetricsPort = 8090
+	explicitFlags = map[string]bool{"vector.port": true, "vector.metricsPort": true}
+	err := ensureAllPortsAvailableOnIP("127.0.0.1", false)
+	if err == nil || !strings.Contains(err.Error(), "shared by explicit flags vector.port and vector.metricsPort") {
+		t.Fatalf("listener conflict not rejected: %v", err)
+	}
+}
diff --git a/weed/command/s3.go b/weed/command/s3.go
index 753cfeacf2..c41d049650 100644
--- a/weed/command/s3.go
+++ b/weed/command/s3.go
@@ -10,7 +10,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"sync"
 	"sync/atomic"
 	"time"
 
@@ -20,10 +19,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/vector_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/iceberg"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/lance"
@@ -37,8 +34,6 @@ import (
 	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
 )
 
 var (
@@ -452,7 +447,7 @@ func (s3opt *S3Options) startS3Server() bool {
 	// backend is reachable — /s3vectors/ in the SigV4 credential scope picks
 	// the vectors router.
 	vectorsHandler := http.Handler(router)
-	if vectorsRouter := s3opt.vectorsRouter(s3ApiServer, grpcDialOption, masterAddresses); vectorsRouter != nil {
+	if vectorsRouter := s3opt.vectorsRouter(s3ApiServer, grpcDialOption, masterAddresses, filerGroup); vectorsRouter != nil {
 		vr, mainRouter := vectorsRouter, router
 		vectorsHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			if isVectorsRequest(r) {
@@ -739,12 +734,12 @@ func (s3opt *S3Options) startLanceServer(listenOnLocalhost bool) *http.Server {
 // vectorsRouter builds the S3 Vectors API router — workers register to the
 // master and the backend resolves them lazily, so a worker that comes up
 // after the gateway still gets picked up on first query.
-func (s3opt *S3Options) vectorsRouter(s3ApiServer *s3api.S3ApiServer, grpcDialOption grpc.DialOption, masterAddresses []pb.ServerAddress) *mux.Router {
+func (s3opt *S3Options) vectorsRouter(s3ApiServer *s3api.S3ApiServer, grpcDialOption grpc.DialOption, masterAddresses []pb.ServerAddress, filerGroup string) *mux.Router {
 	if len(masterAddresses) == 0 {
 		glog.Warningf("S3 Vectors API disabled: no weed-vector registered on master")
 		return nil
 	}
-	backend := &vectorDiscovery{masters: masterAddresses, grpcDialOption: grpcDialOption, discover: discoverVectorWorkers}
+	backend := s3vectors.NewVectorDiscovery(func() []pb.ServerAddress { return masterAddresses }, grpcDialOption, filerGroup)
 	// In-process describe — the router resolves locations and vended
 	// storage options through the catalog directly, never over HTTP. The
 	// lookup itself runs under the trusted identity: the caller's vector
@@ -769,230 +764,6 @@ func (s3opt *S3Options) vectorsRouter(s3ApiServer *s3api.S3ApiServer, grpcDialOp
 	return vectorsRouter
 }
 
-// vectorDiscovery resolves the currently registered weed-vector through the
-// master and caches the connection; workers joining after gateway startup
-// are picked up on first query.
-type vectorDiscovery struct {
-	masters        []pb.ServerAddress
-	grpcDialOption grpc.DialOption
-	discover       func([]pb.ServerAddress, grpc.DialOption) []string
-	mu             sync.Mutex
-	worker         string
-	conn           *grpc.ClientConn
-	client         vector_pb.VectorServiceClient
-	nextResolve    time.Time
-	resolving      chan struct{} // closed when the in-flight resolve lands
-}
-
-// reset drops the cached worker so the next call rediscovers — invoked on
-// transport errors only, and only when the failed client is still the
-// cached one (a concurrent request may have already re-resolved).
-func (d *vectorDiscovery) reset(c vector_pb.VectorServiceClient) {
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	if c != d.client {
-		return
-	}
-	if d.conn != nil {
-		d.conn.Close()
-	}
-	d.worker, d.conn, d.client = "", nil, nil
-	d.nextResolve = time.Time{}
-}
-
-// Only Unavailable proves a transport failure — a request deadline or an
-// application-level error can fire on a healthy connection.
-func isTransportError(err error) bool {
-	return status.Code(err) == codes.Unavailable
-}
-
-// resolve runs without d.mu held so a slow master cannot stall cached-client callers.
-func (d *vectorDiscovery) resolve() {
-	workers := d.discover(d.masters, d.grpcDialOption)
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	// Measured from lookup end so a slow lookup cannot leave the next one already due.
-	defer func() {
-		interval := vectorRecheckInterval
-		if d.client != nil {
-			interval = vectorResolveInterval
-		}
-		d.nextResolve = time.Now().Add(interval)
-		close(d.resolving)
-		d.resolving = nil
-	}()
-	if len(workers) == 0 {
-		return
-	}
-	worker := workers[0]
-	for _, w := range workers {
-		if w == d.worker {
-			worker = w
-			break
-		}
-	}
-	if worker != d.worker {
-		conn, err := pb.GrpcDial(context.Background(), worker, false, d.grpcDialOption)
-		if err != nil {
-			glog.Warningf("S3 Vectors: dial weed-vector %s: %v", worker, err)
-			return
-		}
-		if d.conn != nil {
-			d.conn.Close()
-		}
-		d.worker, d.conn = worker, conn
-		d.client = vector_pb.NewVectorServiceClient(conn)
-	}
-}
-
-// A live client refreshes membership at most every vectorResolveInterval.
-// With no usable backend the recheck is much more frequent: a worker that
-// registers a moment after a lookup must surface inside a caller's retry
-// budget, while an absent one still cannot flood the master with lookups.
-const (
-	vectorResolveInterval = 5 * time.Second
-	vectorRecheckInterval = 500 * time.Millisecond
-)
-
-var errNoVectorWorker = status.Error(codes.Unavailable, "no weed-vector registered on master")
-
-func (d *vectorDiscovery) cli() (vector_pb.VectorServiceClient, error) {
-	d.mu.Lock()
-	if d.resolving == nil && !time.Now().Before(d.nextResolve) {
-		d.resolving = make(chan struct{})
-		go d.resolve()
-	}
-	cli, resolving := d.client, d.resolving
-	d.mu.Unlock()
-	if cli != nil {
-		return cli, nil
-	}
-	// No cached client: wait for the in-flight lookup instead of failing.
-	if resolving != nil {
-		<-resolving
-		d.mu.Lock()
-		cli = d.client
-		d.mu.Unlock()
-		if cli != nil {
-			return cli, nil
-		}
-	}
-	return nil, errNoVectorWorker
-}
-
-func (d *vectorDiscovery) CreateIndex(ctx context.Context, in *vector_pb.CreateIndexRequest, opts ...grpc.CallOption) (resp *vector_pb.CreateIndexResponse, err error) {
-	for attempt := 0; attempt < 2; attempt++ {
-		var c vector_pb.VectorServiceClient
-		if c, err = d.cli(); err != nil {
-			return nil, err
-		}
-		if resp, err = c.CreateIndex(ctx, in, opts...); err == nil {
-			return resp, nil
-		}
-		if !isTransportError(err) {
-			return nil, err
-		}
-		d.reset(c)
-	}
-	return nil, err
-}
-func (d *vectorDiscovery) DropIndex(ctx context.Context, in *vector_pb.DropIndexRequest, opts ...grpc.CallOption) (resp *vector_pb.DropIndexResponse, err error) {
-	for attempt := 0; attempt < 2; attempt++ {
-		var c vector_pb.VectorServiceClient
-		if c, err = d.cli(); err != nil {
-			return nil, err
-		}
-		if resp, err = c.DropIndex(ctx, in, opts...); err == nil {
-			return resp, nil
-		}
-		if !isTransportError(err) {
-			return nil, err
-		}
-		d.reset(c)
-	}
-	return nil, err
-}
-func (d *vectorDiscovery) PutVectors(ctx context.Context, in *vector_pb.PutVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.PutVectorsResponse, err error) {
-	for attempt := 0; attempt < 2; attempt++ {
-		var c vector_pb.VectorServiceClient
-		if c, err = d.cli(); err != nil {
-			return nil, err
-		}
-		if resp, err = c.PutVectors(ctx, in, opts...); err == nil {
-			return resp, nil
-		}
-		if !isTransportError(err) {
-			return nil, err
-		}
-		d.reset(c)
-	}
-	return nil, err
-}
-func (d *vectorDiscovery) GetVectors(ctx context.Context, in *vector_pb.GetVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.GetVectorsResponse, err error) {
-	for attempt := 0; attempt < 2; attempt++ {
-		var c vector_pb.VectorServiceClient
-		if c, err = d.cli(); err != nil {
-			return nil, err
-		}
-		if resp, err = c.GetVectors(ctx, in, opts...); err == nil {
-			return resp, nil
-		}
-		if !isTransportError(err) {
-			return nil, err
-		}
-		d.reset(c)
-	}
-	return nil, err
-}
-func (d *vectorDiscovery) DeleteVectors(ctx context.Context, in *vector_pb.DeleteVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.DeleteVectorsResponse, err error) {
-	for attempt := 0; attempt < 2; attempt++ {
-		var c vector_pb.VectorServiceClient
-		if c, err = d.cli(); err != nil {
-			return nil, err
-		}
-		if resp, err = c.DeleteVectors(ctx, in, opts...); err == nil {
-			return resp, nil
-		}
-		if !isTransportError(err) {
-			return nil, err
-		}
-		d.reset(c)
-	}
-	return nil, err
-}
-func (d *vectorDiscovery) ListVectors(ctx context.Context, in *vector_pb.ListVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.ListVectorsResponse, err error) {
-	for attempt := 0; attempt < 2; attempt++ {
-		var c vector_pb.VectorServiceClient
-		if c, err = d.cli(); err != nil {
-			return nil, err
-		}
-		if resp, err = c.ListVectors(ctx, in, opts...); err == nil {
-			return resp, nil
-		}
-		if !isTransportError(err) {
-			return nil, err
-		}
-		d.reset(c)
-	}
-	return nil, err
-}
-func (d *vectorDiscovery) QueryVectors(ctx context.Context, in *vector_pb.QueryVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.QueryVectorsResponse, err error) {
-	for attempt := 0; attempt < 2; attempt++ {
-		var c vector_pb.VectorServiceClient
-		if c, err = d.cli(); err != nil {
-			return nil, err
-		}
-		if resp, err = c.QueryVectors(ctx, in, opts...); err == nil {
-			return resp, nil
-		}
-		if !isTransportError(err) {
-			return nil, err
-		}
-		d.reset(c)
-	}
-	return nil, err
-}
-
 // isVectorsRequest routes by the SigV4 credential scope — the s3vectors
 // service name lives there, not in the path.
 func isVectorsRequest(r *http.Request) bool {
@@ -1134,32 +905,3 @@ func (v lanceCredentialVendor) VendTableCredentials(ctx context.Context, princip
 // discoverVectorWorkers returns every registered weed-vector address —
 // callers keep their current worker when it is still listed so
 // membership-order changes cannot churn live connections.
-func discoverVectorWorkers(masters []pb.ServerAddress, grpcDialOption grpc.DialOption) []string {
-	for _, master := range masters {
-		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-		conn, err := pb.GrpcDial(ctx, master.ToGrpcAddress(), false, grpcDialOption)
-		if err != nil {
-			cancel()
-			continue
-		}
-		resp, err := master_pb.NewSeaweedClient(conn).ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
-			ClientType: "vector",
-		})
-		conn.Close()
-		cancel()
-		if err != nil {
-			continue
-		}
-		workers := make([]string, 0, len(resp.ClusterNodes))
-		for _, node := range resp.ClusterNodes {
-			glog.V(0).Infof("S3 Vectors: discovered weed-vector at %s via master %s", node.Address, master)
-			workers = append(workers, node.Address)
-		}
-		// A follower can answer with an empty list — workers register with
-		// the leader, so keep looking rather than accept "none."
-		if len(workers) > 0 {
-			return workers
-		}
-	}
-	return nil
-}
diff --git a/weed/command/shell_s3vectors.go b/weed/command/shell_s3vectors.go
new file mode 100644
index 0000000000..ec167dd719
--- /dev/null
+++ b/weed/command/shell_s3vectors.go
@@ -0,0 +1,101 @@
+package command
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"slices"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/lance"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
+	"github.com/seaweedfs/seaweedfs/weed/shell"
+)
+
+func init() { shell.Commands = append(shell.Commands, &commandS3Vectors{}) }
+
+type commandS3Vectors struct{}
+
+func (c *commandS3Vectors) Name() string                 { return "s3vectors" }
+func (c *commandS3Vectors) HasTag(shell.CommandTag) bool { return false }
+func (c *commandS3Vectors) Help() string {
+	return `manage vector buckets, indexes, policies, tags and vectors
+
+s3vectors -operation CreateVectorBucket -request '{"vectorBucketName":"embeddings"}'
+s3vectors -operation ListVectorBuckets
+s3vectors -operation CreateIndex -file index.json
+s3vectors -operation QueryVectors -file query.json
+
+Requests use the S3 Vectors API JSON shape. The shell uses trusted cluster
+credentials. Index and vector operations require a registered vector engine.
+`
+}
+
+func (c *commandS3Vectors) Do(args []string, env *shell.CommandEnv, writer io.Writer) error {
+	flags := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	operation := flags.String("operation", "", "S3 Vectors operation")
+	request := flags.String("request", "", "request JSON")
+	file := flags.String("file", "", "request JSON file")
+	if err := flags.Parse(args); err != nil {
+		return err
+	}
+	if !slices.Contains(s3vectors.Operations(), *operation) {
+		return fmt.Errorf("unknown operation %q; supported: %v", *operation, s3vectors.Operations())
+	}
+	if *request != "" && *file != "" {
+		return fmt.Errorf("use either -request or -file")
+	}
+	body := []byte(*request)
+	if *file != "" {
+		reader, err := os.Open(*file)
+		if err != nil {
+			return err
+		}
+		defer reader.Close()
+		body, err = io.ReadAll(io.LimitReader(reader, s3_constants.S3VectorsMaxRequestBodyBytes+1))
+		if err != nil {
+			return err
+		}
+	}
+	if len(body) == 0 {
+		body = []byte(`{}`)
+	}
+	if len(body) > s3_constants.S3VectorsMaxRequestBodyBytes {
+		return fmt.Errorf("request is too large")
+	}
+	if !json.Valid(body) {
+		return fmt.Errorf("request must be valid JSON")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	backend := s3vectors.NewVectorDiscovery(env.MasterClient.ListCachedMasters, env.GrpcDialOption(), env.MasterClient.FilerGroup)
+	defer backend.Close()
+	server := s3vectors.NewServer(env, nil, backend)
+	server.SetRegion(s3tables.DefaultRegion)
+	server.SetAccountID(s3tables.DefaultAccountID)
+	catalog := lance.NewServer(env, nil)
+	server.SetDescribe(func(ctx context.Context, id []string) (string, map[string]string, error) {
+		r := (&http.Request{Method: http.MethodPost, Header: make(http.Header)}).WithContext(ctx)
+		return catalog.DescribeTableInternal(r, id)
+	})
+	var response json.RawMessage
+	if err := server.Execute(ctx, *operation, json.RawMessage(body), &response, s3_constants.AccountAdminId); err != nil {
+		return err
+	}
+	if len(response) > 0 {
+		var value interface{}
+		if err := json.Unmarshal(response, &value); err != nil {
+			return err
+		}
+		encoder := json.NewEncoder(writer)
+		encoder.SetIndent("", "  ")
+		return encoder.Encode(value)
+	}
+	return nil
+}
diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go
index c2be4829b7..9fb00df081 100644
--- a/weed/s3api/auth_credentials.go
+++ b/weed/s3api/auth_credentials.go
@@ -2748,11 +2748,13 @@ func (iam *IdentityAccessManagement) evaluateAttachedIAMPolicies(r *http.Request
 	if isBucketListingRequest(r, action) {
 		resourceObject = ""
 	}
-	resource := buildResourceARN(bucket, resourceObject)
+	s3Action, resource := resolveS3AuthTarget(action, bucket, resourceObject, r)
 	principal := buildPrincipalARN(identity, r)
-	s3Action := ResolveS3Action(r, string(action), bucket, resourceObject)
 	explicitAllow := false
 	conditions := engine.ExtractConditionValuesFromRequest(r)
+	for key, values := range vectorIAMConditions(r) {
+		conditions[key] = values
+	}
 	for k, v := range policy_engine.ExtractPrincipalVariables(principal) {
 		conditions[k] = v
 	}
@@ -3214,7 +3216,7 @@ func (iam *IdentityAccessManagement) authorizeWithIAM(r *http.Request, identity
 	// The evaluation below consults the bucket-policy: mirror, so the
 	// bucket's lazy load (which backfills that mirror) must happen first -
 	// nothing earlier on a denied request's path would ever trigger it.
-	if iam.primeBucketForIAM != nil && bucket != "" {
+	if iam.primeBucketForIAM != nil && bucket != "" && !strings.HasPrefix(string(action), "s3vectors:") {
 		iam.primeBucketForIAM(bucket)
 	}
 
@@ -3276,6 +3278,9 @@ func (iam *IdentityAccessManagement) authorizeWithIAM(r *http.Request, identity
 // AuthorizeAction applies, so the native-permission floor's explicit-deny
 // check evaluates the same action and resource ARN as the policy engine.
 func resolveS3AuthTarget(action Action, bucket, object string, r *http.Request) (s3Action, resourceArn string) {
+	if strings.HasPrefix(string(action), "s3vectors:") {
+		return string(action), bucket
+	}
 	resourceObjectKey := object
 	if isBucketListingRequest(r, action) {
 		resourceObjectKey = ""
diff --git a/weed/s3api/s3_iam_middleware.go b/weed/s3api/s3_iam_middleware.go
index 6861f48bf4..bb01b72634 100644
--- a/weed/s3api/s3_iam_middleware.go
+++ b/weed/s3api/s3_iam_middleware.go
@@ -257,7 +257,6 @@ func (s3iam *S3IAMIntegration) AuthorizeAction(ctx context.Context, identity *IA
 	}
 
 	// Build resource ARN for the S3 operation
-	resourceArn := buildS3ResourceArn(bucket, resourceObjectKey)
 
 	// Add identity claims to request context for policy variables
 	// Only add claim keys if they don't already exist (to avoid overwriting request-derived context)
@@ -283,7 +282,7 @@ func (s3iam *S3IAMIntegration) AuthorizeAction(ctx context.Context, identity *IA
 	// Determine the specific S3 action based on the HTTP request details. The
 	// prefix promoted into objectKey is not part of the URL; resolve against
 	// the bucket-level object so ?versions and ?uploads keep their own action.
-	specificAction := ResolveS3Action(r, string(action), bucket, resourceObjectKey)
+	specificAction, resourceArn := resolveS3AuthTarget(action, bucket, resourceObjectKey, r)
 
 	// Create action request
 	actionRequest := &integration.ActionRequest{
@@ -397,6 +396,9 @@ func (s3iam *S3IAMIntegration) extractRequestContext(r *http.Request) map[string
 		context["referer"] = referer
 	}
 
+	for key, values := range vectorIAMConditions(r) {
+		context[key] = values
+	}
 	return context
 }
 
diff --git a/weed/s3api/s3api_embedded_iam.go b/weed/s3api/s3api_embedded_iam.go
index 6b5e244469..205824f0df 100644
--- a/weed/s3api/s3api_embedded_iam.go
+++ b/weed/s3api/s3api_embedded_iam.go
@@ -1125,8 +1125,24 @@ func iamPolicyArn(policyName string) string {
 // res[5] contains the bucket and optional path after :::
 func (e *EmbeddedIamApi) getActions(policy *policy_engine.PolicyDocument) ([]string, error) {
 	var actions []string
+	hasVectorPolicy := false
 
 	for _, statement := range policy.Statement {
+		vectorOnly := len(statement.Action.Strings()) > 0
+		for _, action := range statement.Action.Strings() {
+			if !strings.HasPrefix(action, "s3vectors:") {
+				vectorOnly = false
+			}
+		}
+		if vectorOnly && (statement.Effect == policy_engine.PolicyEffectAllow || statement.Effect == policy_engine.PolicyEffectDeny) {
+			for _, resource := range statement.Resource.Strings() {
+				if resource == "*" || strings.HasPrefix(resource, "arn:aws:s3vectors:") {
+					hasVectorPolicy = true
+				}
+			}
+			continue
+		}
+
 		if statement.Effect != policy_engine.PolicyEffectAllow {
 			return nil, fmt.Errorf("not a valid effect: '%s'. Only 'Allow' is possible", statement.Effect)
 		}
@@ -1181,7 +1197,7 @@ func (e *EmbeddedIamApi) getActions(policy *policy_engine.PolicyDocument) ([]str
 		}
 	}
 
-	if len(actions) == 0 {
+	if len(actions) == 0 && !hasVectorPolicy {
 		return nil, fmt.Errorf("no valid actions found in policy document")
 	}
 	return actions, nil
diff --git a/weed/s3api/s3api_status_handlers.go b/weed/s3api/s3api_status_handlers.go
index e868a135ff..a70a883934 100644
--- a/weed/s3api/s3api_status_handlers.go
+++ b/weed/s3api/s3api_status_handlers.go
@@ -1,12 +1,18 @@
 package s3api
 
 import (
+	"encoding/json"
 	"net/http"
 
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 )
 
 func (s3a *S3ApiServer) StatusHandler(w http.ResponseWriter, r *http.Request) {
+	if _, features := r.URL.Query()["features"]; features {
+		body, _ := json.Marshal(map[string]bool{"sseS3": GetSSES3KeyManager().IsConfigured()})
+		s3err.WriteResponse(w, r, http.StatusOK, body, "application/json")
+		return
+	}
 	// write out the response code and content type header
 	s3err.WriteResponse(w, r, http.StatusOK, []byte{}, "")
 }
diff --git a/weed/s3api/s3api_vector_iam.go b/weed/s3api/s3api_vector_iam.go
new file mode 100644
index 0000000000..9c12d53281
--- /dev/null
+++ b/weed/s3api/s3api_vector_iam.go
@@ -0,0 +1,49 @@
+package s3api
+
+import (
+	"context"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+	"net/http"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+)
+
+func (s3a *S3ApiServer) EvaluateIAMPolicy(r *http.Request, action, resource string, policyContext *s3tables.PolicyContext) (allowed, denied bool) {
+	identity, _ := s3_constants.GetIdentityFromContext(r).(*Identity)
+	if identity == nil || s3a.iam == nil {
+		return false, false
+	}
+	conditions := make(map[string][]string)
+	for key, value := range policyContext.ResourceTags {
+		conditions["aws:ResourceTag/"+key] = []string{value}
+	}
+	for key, value := range policyContext.TableBucketTags {
+		conditions["s3vectors:VectorBucketTag/"+key] = []string{value}
+	}
+	for key, value := range policyContext.RequestTags {
+		conditions["aws:RequestTag/"+key] = []string{value}
+	}
+	if len(policyContext.TagKeys) > 0 {
+		conditions["aws:TagKeys"] = policyContext.TagKeys
+	}
+	r = r.WithContext(context.WithValue(r.Context(), vectorIAMConditionsKey{}, conditions))
+
+	result := s3a.iam.evaluateAttachedIAMPolicies(r, identity, Action(action), resource, "")
+	denied = result == attachedIAMPolicyDeny || s3a.iam.isActionExplicitlyDeniedByIAM(r, identity, buildPrincipalARN(identity, r), action, resource)
+	if denied {
+		return false, true
+	}
+	if s3a.iam.authorizationRoute(r, identity) == authorizeViaIAMIntegration {
+		allowed = s3a.iam.VerifyActionPermission(r, identity, Action(action), resource, "") == s3err.ErrNone
+		return allowed, hasSessionToken(r) && !allowed
+	}
+	return result == attachedIAMPolicyAllow, false
+}
+
+type vectorIAMConditionsKey struct{}
+
+func vectorIAMConditions(r *http.Request) map[string][]string {
+	conditions, _ := r.Context().Value(vectorIAMConditionsKey{}).(map[string][]string)
+	return conditions
+}
diff --git a/weed/s3api/s3api_vector_iam_test.go b/weed/s3api/s3api_vector_iam_test.go
new file mode 100644
index 0000000000..945c878d8d
--- /dev/null
+++ b/weed/s3api/s3api_vector_iam_test.go
@@ -0,0 +1,62 @@
+package s3api
+
+import (
+	"fmt"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVectorIAMPolicies(t *testing.T) {
+	const resource = "arn:aws:s3vectors:us-east-1:000000000000:bucket/embeddings/index/docs"
+	for _, action := range []string{"s3vectors:QueryVectors", "s3vectors:*"} {
+		t.Run(action, func(t *testing.T) {
+			iam := &IdentityAccessManagement{isAuthEnabled: true}
+			require.NoError(t, iam.PutPolicy("vectors", fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":%q,"Resource":%q,"Condition":{"StringEquals":{"aws:ResourceTag/team":"search"}}}]}`, action, resource)))
+			identity := &Identity{Name: "reader", PolicyNames: []string{"vectors"}}
+			r := httptest.NewRequest("POST", "/QueryVectors", nil)
+			r = r.WithContext(s3_constants.SetIdentityInContext(r.Context(), identity))
+			server := &S3ApiServer{iam: iam}
+			allowed, denied := server.EvaluateIAMPolicy(r, "s3vectors:QueryVectors", resource, &s3tables.PolicyContext{ResourceTags: map[string]string{"team": "search"}})
+			require.True(t, allowed)
+			require.False(t, denied)
+			allowed, denied = server.EvaluateIAMPolicy(r, "s3vectors:QueryVectors", resource, &s3tables.PolicyContext{ResourceTags: map[string]string{"team": "other"}})
+			require.False(t, allowed)
+			require.False(t, denied)
+			allowed, _ = server.EvaluateIAMPolicy(r, "s3vectors:QueryVectors", resource+"-other", &s3tables.PolicyContext{ResourceTags: map[string]string{"team": "search"}})
+			require.False(t, allowed)
+			require.NoError(t, iam.PutPolicy("deny", `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"s3vectors:QueryVectors","Resource":"*"}]}`))
+			identity.PolicyNames = append(identity.PolicyNames, "deny")
+			allowed, denied = server.EvaluateIAMPolicy(r, "s3vectors:QueryVectors", resource, &s3tables.PolicyContext{ResourceTags: map[string]string{"team": "search"}})
+			require.False(t, allowed)
+			require.True(t, denied)
+		})
+	}
+}
+
+func TestEmbeddedIAMAcceptsVectorPoliciesWithoutFlattening(t *testing.T) {
+	api := &EmbeddedIamApi{}
+	for _, effect := range []string{"Allow", "Deny"} {
+		raw := fmt.Sprintf(`{"Statement":[{"Effect":%q,"Action":"s3vectors:*","Resource":"arn:aws:s3vectors:us-east-1:000000000000:bucket/embeddings/*"}]}`, effect)
+		policy, err := api.GetPolicyDocument(&raw)
+		require.NoError(t, err)
+		actions, err := api.getActions(&policy)
+		require.NoError(t, err)
+		require.Empty(t, actions)
+	}
+}
+
+func TestVectorIAMBucketTagDeny(t *testing.T) {
+	iam := &IdentityAccessManagement{isAuthEnabled: true}
+	require.NoError(t, iam.PutPolicy("vectors", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3vectors:*","Resource":"*"},{"Effect":"Deny","Action":"s3vectors:QueryVectors","Resource":"*","Condition":{"StringEquals":{"s3vectors:VectorBucketTag/classification":"private"}}}]}`))
+	identity := &Identity{Name: "reader", PolicyNames: []string{"vectors"}}
+	r := httptest.NewRequest("POST", "/QueryVectors", nil)
+	r = r.WithContext(s3_constants.SetIdentityInContext(r.Context(), identity))
+	server := &S3ApiServer{iam: iam}
+	allowed, denied := server.EvaluateIAMPolicy(r, "s3vectors:QueryVectors", "arn:aws:s3vectors:us-east-1:000000000000:bucket/embeddings/index/docs", &s3tables.PolicyContext{TableBucketTags: map[string]string{"classification": "private"}})
+	require.False(t, allowed)
+	require.True(t, denied)
+}
diff --git a/weed/s3api/s3vectors/create_recovery_test.go b/weed/s3api/s3vectors/create_recovery_test.go
new file mode 100644
index 0000000000..74490ab1e0
--- /dev/null
+++ b/weed/s3api/s3vectors/create_recovery_test.go
@@ -0,0 +1,293 @@
+package s3vectors
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/vector_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type failingCreateVectors struct {
+	*fakeVectors
+	err      error
+	onCreate func()
+}
+
+func (f *failingCreateVectors) CreateIndex(_ context.Context, request *vector_pb.CreateIndexRequest, _ ...grpc.CallOption) (*vector_pb.CreateIndexResponse, error) {
+	f.created = append(f.created, request)
+	err := f.err
+	if f.onCreate != nil {
+		f.onCreate()
+	}
+	return nil, err
+}
+
+func TestCreateIndexPreservesAnUnknownOutcome(t *testing.T) {
+	for _, code := range []codes.Code{codes.Unavailable, codes.DeadlineExceeded, codes.Canceled, codes.Internal, codes.InvalidArgument} {
+		t.Run(code.String(), func(t *testing.T) {
+			fs := s3tablestest.Start(t)
+			engine := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(code, "response lost")}
+			server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
+			postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+			result := postJSON(t, server.handleCreateIndex, &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "cosine"})
+			if result.Code == http.StatusOK {
+				t.Fatal("failed engine reported success")
+			}
+			got := postJSON(t, server.handleGetIndex, &GetIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx"})
+			want := http.StatusOK
+			if code == codes.InvalidArgument {
+				want = http.StatusNotFound
+			}
+			if got.Code != want {
+				t.Fatalf("catalog after %s: %d %s", code, got.Code, got.Body)
+			}
+		})
+	}
+}
+
+func TestCreateIndexRetainsTheWriteErrorIfRediscoveryFails(t *testing.T) {
+	engine := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(codes.Unavailable, "response lost after commit")}
+	discovery := &vectorDiscovery{client: engine, nextResolve: time.Now().Add(time.Hour), discover: func([]pb.ServerAddress, grpc.DialOption) []string { return nil }}
+	_, err := discovery.CreateIndex(t.Context(), &vector_pb.CreateIndexRequest{})
+	if status.Convert(err).Message() != "response lost after commit" || len(engine.created) != 1 {
+		t.Fatalf("ambiguous result replaced by lookup failure: %v", err)
+	}
+}
+
+func TestCreateIndexResumesAnUnknownOutcome(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	engine := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(codes.Unavailable, "request lost")}
+	server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
+	postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+	request := &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "cosine",
+		MetadataConfiguration: &MetadataConfiguration{NonFilterableMetadataKeys: []string{"b", "a"}}, Tags: map[string]string{"team": "search"}}
+	if result := postJSON(t, server.handleCreateIndex, request); result.Code == http.StatusOK {
+		t.Fatal("unavailable engine reported success")
+	}
+	before, err := server.indexRecord(t.Context(), "vec-bucket", "idx")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, change := range []func(*CreateIndexRequest){
+		func(r *CreateIndexRequest) { r.Dimension++ },
+		func(r *CreateIndexRequest) { r.DistanceMetric = "euclidean" },
+		func(r *CreateIndexRequest) { r.MetadataConfiguration = nil },
+	} {
+		other := *request
+		change(&other)
+		if result := postJSON(t, server.handleCreateIndex, &other); result.Code != http.StatusConflict {
+			t.Fatalf("incompatible retry: %d %s", result.Code, result.Body)
+		}
+	}
+	if len(engine.created) != 1 {
+		t.Fatal("incompatible retry reached the engine")
+	}
+	engine.err = status.Error(codes.InvalidArgument, "retry refused")
+	postJSON(t, server.handleCreateIndex, request)
+	if _, err := server.indexRecord(t.Context(), "vec-bucket", "idx"); err != nil {
+		t.Fatalf("retry removed an uncertain dataset's catalog: %v", err)
+	}
+	engine.err = nil
+	request.MetadataConfiguration.NonFilterableMetadataKeys = []string{"a", "b"}
+	if result := postJSON(t, server.handleCreateIndex, request); result.Code != http.StatusOK {
+		t.Fatalf("resume: %d %s", result.Code, result.Body)
+	}
+	after, err := server.indexRecord(t.Context(), "vec-bucket", "idx")
+	if err != nil || recordNonce(before) != recordNonce(after) {
+		t.Fatalf("resume replaced the catalog: %v", err)
+	}
+	if result := postJSON(t, server.handleCreateIndex, request); result.Code != http.StatusConflict {
+		t.Fatalf("completed index was initialized again: %d %s", result.Code, result.Body)
+	}
+	result := postJSON(t, server.handleListTagsForResource, &ListTagsForResourceRequest{ResourceArn: server.indexArn("vec-bucket", "idx")})
+	var tags ListTagsForResourceResponse
+	if err := json.Unmarshal(result.Body.Bytes(), &tags); err != nil || tags.Tags["team"] != "search" {
+		t.Fatalf("resume lost tags: %s (%v)", result.Body, err)
+	}
+}
+
+func TestCreateIndexRollbackPreservesACompletedRetry(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	engine := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(codes.InvalidArgument, "first attempt failed")}
+	server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
+	postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+	request := &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "cosine"}
+	engine.onCreate = func() {
+		engine.onCreate = nil
+		engine.err = nil
+		if result := postJSON(t, server.handleCreateIndex, request); result.Code != http.StatusOK {
+			t.Fatalf("concurrent retry: %d %s", result.Code, result.Body)
+		}
+	}
+	if result := postJSON(t, server.handleCreateIndex, request); result.Code == http.StatusOK {
+		t.Fatal("first attempt reported success")
+	}
+	if _, err := server.indexRecord(t.Context(), "vec-bucket", "idx"); err != nil {
+		t.Fatalf("rollback removed the completed retry: %v", err)
+	}
+	if result := postJSON(t, server.handleCreateIndex, request); result.Code != http.StatusConflict {
+		t.Fatalf("completed retry lost its state: %d %s", result.Code, result.Body)
+	}
+}
+
+func TestCreateIndexRetainsAnUnknownOutcomeAfterARefusedRetry(t *testing.T) {
+	for _, code := range []codes.Code{codes.PermissionDenied, codes.FailedPrecondition} {
+		t.Run(code.String(), func(t *testing.T) {
+			fs := s3tablestest.Start(t)
+			first := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(codes.Unavailable, "response lost after commit")}
+			second := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(code, "retry refused")}
+			discovery := &vectorDiscovery{client: first, nextResolve: time.Now().Add(time.Hour)}
+			discovery.discover = func([]pb.ServerAddress, grpc.DialOption) []string {
+				discovery.mu.Lock()
+				discovery.client = second
+				discovery.mu.Unlock()
+				return nil
+			}
+			defer discovery.Close()
+			server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, discovery)
+			postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+			result := postJSON(t, server.handleCreateIndex, &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "cosine"})
+			if result.Code != http.StatusServiceUnavailable || len(first.created) != 1 || len(second.created) != 1 {
+				t.Fatalf("unknown outcome lost: %d %s (attempts %d, %d)", result.Code, result.Body, len(first.created), len(second.created))
+			}
+			if _, err := server.indexRecord(t.Context(), "vec-bucket", "idx"); err != nil {
+				t.Fatalf("refused retry erased a possibly committed dataset: %v", err)
+			}
+		})
+	}
+}
+
+type concurrentCreateVectors struct {
+	vector_pb.VectorServiceClient
+	create func() (*vector_pb.CreateIndexResponse, error)
+}
+
+func (f *concurrentCreateVectors) CreateIndex(context.Context, *vector_pb.CreateIndexRequest, ...grpc.CallOption) (*vector_pb.CreateIndexResponse, error) {
+	return f.create()
+}
+
+func TestCreateIndexRollbackPreservesAnActiveRetry(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	firstStarted, retryStarted, finishRetry := make(chan struct{}), make(chan struct{}), make(chan struct{})
+	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+	defer cancel()
+	var calls atomic.Int32
+	engine := &concurrentCreateVectors{create: func() (*vector_pb.CreateIndexResponse, error) {
+		if calls.Add(1) == 1 {
+			close(firstStarted)
+			select {
+			case <-retryStarted:
+				return nil, status.Error(codes.InvalidArgument, "first attempt refused")
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			}
+		}
+		close(retryStarted)
+		select {
+		case <-finishRetry:
+			return &vector_pb.CreateIndexResponse{}, nil
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}}
+	server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
+	postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+	request := &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "cosine"}
+	first, retry := make(chan *httptest.ResponseRecorder, 1), make(chan *httptest.ResponseRecorder, 1)
+	go func() { first <- postJSON(t, server.handleCreateIndex, request) }()
+	select {
+	case <-firstStarted:
+	case <-ctx.Done():
+		t.Fatal("first create did not reach the engine")
+	}
+	go func() { retry <- postJSON(t, server.handleCreateIndex, request) }()
+	if result := <-first; result.Code == http.StatusOK {
+		t.Fatal("first attempt reported success")
+	}
+	if _, err := server.indexRecord(t.Context(), "vec-bucket", "idx"); err != nil {
+		t.Fatalf("rollback removed an active retry: %v", err)
+	}
+	close(finishRetry)
+	if result := <-retry; result.Code != http.StatusOK {
+		t.Fatalf("retry could not finish: %d %s", result.Code, result.Body)
+	}
+}
+
+func TestCreateIndexResumesWithTheDefaultMetric(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	engine := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(codes.Unavailable, "request lost")}
+	server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
+	postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+	request := &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3}
+	postJSON(t, server.handleCreateIndex, request)
+	engine.err = nil
+	if result := postJSON(t, server.handleCreateIndex, request); result.Code != http.StatusOK {
+		t.Fatalf("default-metric recovery: %d %s", result.Code, result.Body)
+	}
+	if len(engine.created) != 2 || engine.created[1].DistanceMetric != vector_pb.DistanceMetric_L2 {
+		t.Fatal("recovery changed the default metric")
+	}
+}
+
+func TestCreateIndexRetryPreservesDuplicateMetadataKeys(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	engine := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(codes.Unavailable, "response lost")}
+	server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
+	postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+	request := &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "cosine",
+		MetadataConfiguration: &MetadataConfiguration{NonFilterableMetadataKeys: []string{"b", "a", "a"}}}
+	postJSON(t, server.handleCreateIndex, request)
+	engine.err = nil
+	if result := postJSON(t, server.handleCreateIndex, request); result.Code != http.StatusOK {
+		t.Fatalf("duplicate-key recovery: %d %s", result.Code, result.Body)
+	}
+	if len(engine.created) != 2 {
+		t.Fatalf("engine calls: %d", len(engine.created))
+	}
+	for _, call := range engine.created {
+		keys := slices.Clone(call.NonFilterableMetadataKeys)
+		slices.Sort(keys)
+		if !slices.Equal(keys, []string{"a", "a", "b"}) {
+			t.Fatalf("initialization changed metadata keys: %q", keys)
+		}
+	}
+}
+
+func TestCreateIndexDoesNotInitializeAConcurrentCatalogCreate(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	engine := &fakeVectors{}
+	server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
+	postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+	postJSON(t, server.handleCreateIndex, &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "seed", DataType: "float32", Dimension: 7, DistanceMetric: "cosine"})
+	dir, name := splitEntryPath(s3tables.GetTablePath("vec-bucket", indexNamespace, "idx"))
+	seed := fs.Get(dir, "seed")
+	var lookups int
+	fs.OnLookup = func(d, n string) {
+		if d == dir && n == name {
+			lookups++
+			if lookups == 2 {
+				fs.Put(dir, name, seed.Extended)
+			}
+		}
+	}
+	result := postJSON(t, server.handleCreateIndex, &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "cosine"})
+	if result.Code != http.StatusConflict || len(engine.created) != 1 {
+		t.Fatalf("initialized a competing record: %d %s (%d engine calls)", result.Code, result.Body, len(engine.created))
+	}
+	record, err := server.indexRecord(t.Context(), "vec-bucket", "idx")
+	if err != nil || indexFromRecord(server, "vec-bucket", "idx", record).Dimension != 7 {
+		t.Fatalf("changed the competing record: %v", err)
+	}
+}
diff --git a/weed/s3api/s3vectors/discovery.go b/weed/s3api/s3vectors/discovery.go
new file mode 100644
index 0000000000..7edd3e43ef
--- /dev/null
+++ b/weed/s3api/s3vectors/discovery.go
@@ -0,0 +1,358 @@
+package s3vectors
+
+import (
+	"context"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/vector_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"sync"
+	"time"
+)
+
+func NewVectorDiscovery(masters func() []pb.ServerAddress, dialOption grpc.DialOption, filerGroup string) *vectorDiscovery {
+	return &vectorDiscovery{masters: masters, grpcDialOption: dialOption, discover: func(m []pb.ServerAddress, opt grpc.DialOption) []string {
+		return discoverVectorWorkers(m, opt, filerGroup)
+	}}
+}
+
+type vectorDiscovery struct {
+	masters        func() []pb.ServerAddress
+	grpcDialOption grpc.DialOption
+	discover       func([]pb.ServerAddress, grpc.DialOption) []string
+	mu             sync.Mutex
+	closed         bool
+	worker         string
+	failed         string
+	active         map[vector_pb.VectorServiceClient]int
+	retired        map[vector_pb.VectorServiceClient]*grpc.ClientConn
+	conn           *grpc.ClientConn
+	client         vector_pb.VectorServiceClient
+	nextResolve    time.Time
+	resolving      chan struct{} // closed when the in-flight resolve lands
+}
+
+func (d *vectorDiscovery) Close() error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	d.closed = true
+	d.retire()
+	d.worker, d.conn, d.client = "", nil, nil
+	return nil
+}
+
+func (d *vectorDiscovery) reset(c vector_pb.VectorServiceClient) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if c != d.client {
+		return
+	}
+	d.failed = d.worker
+	d.retire()
+	d.worker, d.conn, d.client = "", nil, nil
+	d.nextResolve = time.Time{}
+}
+
+func (d *vectorDiscovery) retire() {
+	if d.conn == nil {
+		return
+	}
+	if d.active[d.client] == 0 {
+		_ = d.conn.Close()
+		return
+	}
+	if d.retired == nil {
+		d.retired = make(map[vector_pb.VectorServiceClient]*grpc.ClientConn)
+	}
+	d.retired[d.client] = d.conn
+}
+
+func (d *vectorDiscovery) release(c vector_pb.VectorServiceClient) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	d.active[c]--
+	if d.active[c] == 0 {
+		delete(d.active, c)
+		if conn := d.retired[c]; conn != nil {
+			_ = conn.Close()
+			delete(d.retired, c)
+		}
+	}
+}
+
+func isTransportError(err error) bool {
+	return status.Code(err) == codes.Unavailable
+}
+
+func (d *vectorDiscovery) resolve() {
+	var masters []pb.ServerAddress
+	if d.masters != nil {
+		masters = d.masters()
+	}
+	workers := d.discover(masters, d.grpcDialOption)
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	// Measured from lookup end so a slow lookup cannot leave the next one already due.
+	defer func() {
+		interval := vectorRecheckInterval
+		if d.client != nil {
+			interval = vectorResolveInterval
+		}
+		d.nextResolve = time.Now().Add(interval)
+		close(d.resolving)
+		d.resolving = nil
+	}()
+	if d.closed || len(workers) == 0 {
+		return
+	}
+	worker := workers[0]
+	for _, w := range workers {
+		if w != d.failed {
+			worker = w
+			break
+		}
+	}
+	for _, w := range workers {
+		if w == d.worker && w != d.failed {
+			worker = w
+			break
+		}
+	}
+	if worker != d.worker {
+		conn, err := pb.GrpcDial(context.Background(), worker, false, d.grpcDialOption)
+		if err != nil {
+			glog.Warningf("S3 Vectors: dial weed-vector %s: %v", worker, err)
+			return
+		}
+		d.retire()
+		d.worker, d.conn = worker, conn
+		d.client = vector_pb.NewVectorServiceClient(conn)
+	}
+}
+
+const (
+	vectorResolveInterval = 5 * time.Second
+	vectorRecheckInterval = 500 * time.Millisecond
+)
+
+var errNoVectorWorker = status.Error(codes.Unavailable, "no weed-vector registered on master")
+
+func (d *vectorDiscovery) cli(ctx context.Context) (vector_pb.VectorServiceClient, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, status.FromContextError(err).Err()
+	}
+	d.mu.Lock()
+	if d.closed {
+		d.mu.Unlock()
+		return nil, status.Error(codes.Canceled, "vector discovery closed")
+	}
+	if d.resolving == nil && !time.Now().Before(d.nextResolve) {
+		d.resolving = make(chan struct{})
+		go d.resolve()
+	}
+	if d.client == nil && d.resolving != nil {
+		resolving := d.resolving
+		d.mu.Unlock()
+		select {
+		case <-resolving:
+		case <-ctx.Done():
+			return nil, status.FromContextError(ctx.Err()).Err()
+		}
+		d.mu.Lock()
+	}
+	defer d.mu.Unlock()
+	if d.closed {
+		return nil, status.Error(codes.Canceled, "vector discovery closed")
+	}
+	if d.client == nil {
+		return nil, errNoVectorWorker
+	}
+	if d.active == nil {
+		d.active = make(map[vector_pb.VectorServiceClient]int)
+	}
+	d.active[d.client]++
+	return d.client, nil
+}
+
+func (d *vectorDiscovery) CreateIndex(ctx context.Context, in *vector_pb.CreateIndexRequest, opts ...grpc.CallOption) (resp *vector_pb.CreateIndexResponse, err error) {
+	var firstErr error
+	for attempt := 0; attempt < 2; attempt++ {
+		var c vector_pb.VectorServiceClient
+		var resolveErr error
+		if c, resolveErr = d.cli(ctx); resolveErr != nil {
+			if attempt > 0 {
+				return nil, firstErr
+			}
+			return nil, resolveErr
+		}
+		resp, err = c.CreateIndex(ctx, in, opts...)
+		if isTransportError(err) {
+			d.reset(c)
+		}
+		d.release(c)
+		if err == nil {
+			return resp, nil
+		}
+		if firstErr == nil {
+			firstErr = err
+		}
+		if !isTransportError(err) {
+			return nil, firstErr
+		}
+	}
+	return nil, firstErr
+}
+func (d *vectorDiscovery) DropIndex(ctx context.Context, in *vector_pb.DropIndexRequest, opts ...grpc.CallOption) (resp *vector_pb.DropIndexResponse, err error) {
+	for attempt := 0; attempt < 2; attempt++ {
+		var c vector_pb.VectorServiceClient
+		if c, err = d.cli(ctx); err != nil {
+			return nil, err
+		}
+		resp, err = c.DropIndex(ctx, in, opts...)
+		if isTransportError(err) {
+			d.reset(c)
+		}
+		d.release(c)
+		if err == nil {
+			return resp, nil
+		}
+		if !isTransportError(err) {
+			return nil, err
+		}
+	}
+	return nil, err
+}
+func (d *vectorDiscovery) PutVectors(ctx context.Context, in *vector_pb.PutVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.PutVectorsResponse, err error) {
+	for attempt := 0; attempt < 2; attempt++ {
+		var c vector_pb.VectorServiceClient
+		if c, err = d.cli(ctx); err != nil {
+			return nil, err
+		}
+		resp, err = c.PutVectors(ctx, in, opts...)
+		if isTransportError(err) {
+			d.reset(c)
+		}
+		d.release(c)
+		if err == nil {
+			return resp, nil
+		}
+		if !isTransportError(err) {
+			return nil, err
+		}
+	}
+	return nil, err
+}
+func (d *vectorDiscovery) GetVectors(ctx context.Context, in *vector_pb.GetVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.GetVectorsResponse, err error) {
+	for attempt := 0; attempt < 2; attempt++ {
+		var c vector_pb.VectorServiceClient
+		if c, err = d.cli(ctx); err != nil {
+			return nil, err
+		}
+		resp, err = c.GetVectors(ctx, in, opts...)
+		if isTransportError(err) {
+			d.reset(c)
+		}
+		d.release(c)
+		if err == nil {
+			return resp, nil
+		}
+		if !isTransportError(err) {
+			return nil, err
+		}
+	}
+	return nil, err
+}
+func (d *vectorDiscovery) DeleteVectors(ctx context.Context, in *vector_pb.DeleteVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.DeleteVectorsResponse, err error) {
+	for attempt := 0; attempt < 2; attempt++ {
+		var c vector_pb.VectorServiceClient
+		if c, err = d.cli(ctx); err != nil {
+			return nil, err
+		}
+		resp, err = c.DeleteVectors(ctx, in, opts...)
+		if isTransportError(err) {
+			d.reset(c)
+		}
+		d.release(c)
+		if err == nil {
+			return resp, nil
+		}
+		if !isTransportError(err) {
+			return nil, err
+		}
+	}
+	return nil, err
+}
+func (d *vectorDiscovery) ListVectors(ctx context.Context, in *vector_pb.ListVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.ListVectorsResponse, err error) {
+	for attempt := 0; attempt < 2; attempt++ {
+		var c vector_pb.VectorServiceClient
+		if c, err = d.cli(ctx); err != nil {
+			return nil, err
+		}
+		resp, err = c.ListVectors(ctx, in, opts...)
+		if isTransportError(err) {
+			d.reset(c)
+		}
+		d.release(c)
+		if err == nil {
+			return resp, nil
+		}
+		if !isTransportError(err) {
+			return nil, err
+		}
+	}
+	return nil, err
+}
+func (d *vectorDiscovery) QueryVectors(ctx context.Context, in *vector_pb.QueryVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.QueryVectorsResponse, err error) {
+	for attempt := 0; attempt < 2; attempt++ {
+		var c vector_pb.VectorServiceClient
+		if c, err = d.cli(ctx); err != nil {
+			return nil, err
+		}
+		resp, err = c.QueryVectors(ctx, in, opts...)
+		if isTransportError(err) {
+			d.reset(c)
+		}
+		d.release(c)
+		if err == nil {
+			return resp, nil
+		}
+		if !isTransportError(err) {
+			return nil, err
+		}
+	}
+	return nil, err
+}
+
+func discoverVectorWorkers(masters []pb.ServerAddress, grpcDialOption grpc.DialOption, filerGroup string) []string {
+	for _, master := range masters {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		conn, err := pb.GrpcDial(ctx, master.ToGrpcAddress(), false, grpcDialOption)
+		if err != nil {
+			cancel()
+			continue
+		}
+		resp, err := master_pb.NewSeaweedClient(conn).ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
+			ClientType: "vector",
+			FilerGroup: filerGroup,
+		})
+		conn.Close()
+		cancel()
+		if err != nil {
+			continue
+		}
+		workers := make([]string, 0, len(resp.ClusterNodes))
+		for _, node := range resp.ClusterNodes {
+			glog.V(0).Infof("S3 Vectors: discovered weed-vector at %s via master %s", node.Address, master)
+			workers = append(workers, node.Address)
+		}
+		// A follower can answer with an empty list — workers register with
+		// the leader, so keep looking rather than accept "none."
+		if len(workers) > 0 {
+			return workers
+		}
+	}
+	return nil
+}
diff --git a/weed/command/s3_vector_discovery_test.go b/weed/s3api/s3vectors/discovery_test.go
similarity index 63%
rename from weed/command/s3_vector_discovery_test.go
rename to weed/s3api/s3vectors/discovery_test.go
index 2a516e3599..008a430c4e 100644
--- a/weed/command/s3_vector_discovery_test.go
+++ b/weed/s3api/s3vectors/discovery_test.go
@@ -1,6 +1,7 @@
-package command
+package s3vectors
 
 import (
+	"context"
 	"fmt"
 	"sync/atomic"
 	"testing"
@@ -26,10 +27,10 @@ func TestVectorDiscoveryRechecksRegistration(t *testing.T) {
 		},
 	}
 
-	if _, err := d.cli(); status.Code(err) != codes.Unavailable {
+	if _, err := d.cli(t.Context()); status.Code(err) != codes.Unavailable {
 		t.Fatalf("no registered worker: expected Unavailable, got %v", err)
 	}
-	if _, err := d.cli(); status.Code(err) != codes.Unavailable {
+	if _, err := d.cli(t.Context()); status.Code(err) != codes.Unavailable {
 		t.Fatalf("no registered worker: expected Unavailable, got %v", err)
 	}
 	if got := atomic.LoadInt32(&calls); got != 1 {
@@ -46,7 +47,7 @@ func TestVectorDiscoveryRechecksRegistration(t *testing.T) {
 	}
 	d.grpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
 
-	if _, err := d.cli(); err != nil {
+	if _, err := d.cli(t.Context()); err != nil {
 		t.Fatalf("worker registered: expected a client, got %v", err)
 	}
 	if got := atomic.LoadInt32(&calls); got != 2 {
@@ -54,7 +55,7 @@ func TestVectorDiscoveryRechecksRegistration(t *testing.T) {
 	}
 
 	// A live client rides out the refresh interval without rediscovery.
-	if _, err := d.cli(); err != nil {
+	if _, err := d.cli(t.Context()); err != nil {
 		t.Fatalf("cached client rejected: %v", err)
 	}
 	if got := atomic.LoadInt32(&calls); got != 2 {
@@ -77,10 +78,10 @@ func TestVectorDiscoveryThrottlesFromLookupEnd(t *testing.T) {
 		},
 	}
 
-	if _, err := d.cli(); status.Code(err) != codes.Unavailable {
+	if _, err := d.cli(t.Context()); status.Code(err) != codes.Unavailable {
 		t.Fatalf("no registered worker: expected Unavailable, got %v", err)
 	}
-	if _, err := d.cli(); status.Code(err) != codes.Unavailable {
+	if _, err := d.cli(t.Context()); status.Code(err) != codes.Unavailable {
 		t.Fatalf("no registered worker: expected Unavailable, got %v", err)
 	}
 	if got := atomic.LoadInt32(&calls); got != 1 {
@@ -95,7 +96,7 @@ func TestVectorDiscoveryThrottlesFromLookupEnd(t *testing.T) {
 		d.mu.Lock()
 		d.nextResolve = time.Time{}
 		d.mu.Unlock()
-		if _, err := d.cli(); err != nil {
+		if _, err := d.cli(t.Context()); err != nil {
 			t.Fatalf("worker registered: expected a client, got %v", err)
 		}
 		waitResolved(d)
@@ -126,7 +127,7 @@ func TestVectorDiscoveryLookupDoesNotBlockCachedClient(t *testing.T) {
 			return []string{"127.0.0.1:9301"}
 		},
 	}
-	cached, err := d.cli()
+	cached, err := d.cli(t.Context())
 	if err != nil {
 		t.Fatalf("worker registered: expected a client, got %v", err)
 	}
@@ -147,7 +148,7 @@ func TestVectorDiscoveryLookupDoesNotBlockCachedClient(t *testing.T) {
 	const callers = 8
 	got := make(chan error, callers+1)
 	call := func() {
-		c, err := d.cli()
+		c, err := d.cli(t.Context())
 		if err == nil && c != cached {
 			err = fmt.Errorf("got a different client")
 		}
@@ -195,7 +196,7 @@ func TestVectorDiscoveryUncachedCallersShareLookup(t *testing.T) {
 	got := make(chan error, callers)
 	for i := 0; i < callers; i++ {
 		go func() {
-			_, err := d.cli()
+			_, err := d.cli(t.Context())
 			got <- err
 		}()
 	}
@@ -219,7 +220,7 @@ func TestVectorDiscoveryKeepsConnOnDialError(t *testing.T) {
 			return []string{"127.0.0.1:9301"}
 		},
 	}
-	cached, err := d.cli()
+	cached, err := d.cli(t.Context())
 	if err != nil {
 		t.Fatalf("worker registered: expected a client, got %v", err)
 	}
@@ -232,7 +233,7 @@ func TestVectorDiscoveryKeepsConnOnDialError(t *testing.T) {
 	d.mu.Lock()
 	d.nextResolve = time.Time{}
 	d.mu.Unlock()
-	if c, err := d.cli(); err != nil || c != cached {
+	if c, err := d.cli(t.Context()); err != nil || c != cached {
 		t.Fatalf("cached client: got %v, %v", c, err)
 	}
 	waitResolved(d)
@@ -243,3 +244,109 @@ func TestVectorDiscoveryKeepsConnOnDialError(t *testing.T) {
 		t.Fatalf("failed dial dropped the cached connection")
 	}
 }
+
+func TestVectorDiscoveryDrainsRetiredConnections(t *testing.T) {
+	d := &vectorDiscovery{grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), discover: func([]pb.ServerAddress, grpc.DialOption) []string { return []string{"127.0.0.1:9301"} }}
+	first, err := d.cli(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := d.cli(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	conn := d.conn
+	d.reset(first)
+	d.release(first)
+	if conn.GetState() == connectivity.Shutdown {
+		t.Fatal("active RPC lost its connection")
+	}
+	d.release(second)
+	if conn.GetState() != connectivity.Shutdown {
+		t.Fatal("retired connection was not closed after the last RPC")
+	}
+}
+
+func TestVectorDiscoveryWaitingRespectsCancellation(t *testing.T) {
+	entered, release := make(chan struct{}), make(chan struct{})
+	d := &vectorDiscovery{discover: func([]pb.ServerAddress, grpc.DialOption) []string { close(entered); <-release; return nil }}
+	ctx, cancel := context.WithCancel(t.Context())
+	result := make(chan error, 1)
+	go func() { _, err := d.cli(ctx); result <- err }()
+	<-entered
+	cancel()
+	select {
+	case err := <-result:
+		if status.Code(err) != codes.Canceled {
+			t.Fatal(err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("request remained blocked after cancellation")
+	}
+	close(release)
+	waitResolved(d)
+}
+
+func TestVectorDiscoveryRetriesAnotherWorker(t *testing.T) {
+	d := &vectorDiscovery{grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), discover: func([]pb.ServerAddress, grpc.DialOption) []string {
+		return []string{"127.0.0.1:9301", "127.0.0.1:9302"}
+	}}
+	first, err := d.cli(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	d.reset(first)
+	d.release(first)
+	second, err := d.cli(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer d.release(second)
+	if d.worker != "127.0.0.1:9302" {
+		t.Fatalf("retry selected failed worker: %s", d.worker)
+	}
+}
+
+func TestVectorDiscoveryCloseDuringLookup(t *testing.T) {
+	entered, release := make(chan struct{}), make(chan struct{})
+	d := &vectorDiscovery{grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), discover: func([]pb.ServerAddress, grpc.DialOption) []string {
+		close(entered)
+		<-release
+		return []string{"127.0.0.1:9301"}
+	}}
+	done := make(chan error, 1)
+	go func() { _, err := d.cli(t.Context()); done <- err }()
+	<-entered
+	_ = d.Close()
+	close(release)
+	if err := <-done; status.Code(err) != codes.Canceled {
+		t.Fatal(err)
+	}
+	if d.conn != nil {
+		t.Fatal("closed discovery opened a connection")
+	}
+}
+
+func TestVectorDiscoveryRefreshesMasterSnapshot(t *testing.T) {
+	masters := []pb.ServerAddress{}
+	d := NewVectorDiscovery(func() []pb.ServerAddress { return masters }, grpc.WithTransportCredentials(insecure.NewCredentials()), "")
+	d.discover = func(current []pb.ServerAddress, _ grpc.DialOption) []string {
+		if len(current) > 0 {
+			return []string{"127.0.0.1:9301"}
+		}
+		return nil
+	}
+	if _, err := d.cli(t.Context()); status.Code(err) != codes.Unavailable {
+		t.Fatal(err)
+	}
+	masters = []pb.ServerAddress{"master:9333"}
+	d.mu.Lock()
+	d.nextResolve = time.Time{}
+	d.mu.Unlock()
+	client, err := d.cli(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	d.release(client)
+	_ = d.Close()
+}
diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go
index f9f5b75cea..17ffa86548 100644
--- a/weed/s3api/s3vectors/handlers.go
+++ b/weed/s3api/s3vectors/handlers.go
@@ -295,7 +295,11 @@ func (s *Server) handleCreateVectorBucket(w http.ResponseWriter, r *http.Request
 	// catalog does for CreateTableBucket.
 	principal := s.tables.PrincipalFor(r)
 	allowed := func(action, bucket string) bool {
-		return s3tables.CheckPermissionWithContext(action, principal, s.tables.PrincipalFor(nil), "", "", s.tables.PolicyContext(r, bucket))
+		resource := "*"
+		if bucket != "" {
+			resource = s.vectorBucketArn(bucket)
+		}
+		return s.checkPermission(r, action, principal, s.tables.PrincipalFor(nil), "", resource, s.policyContext(r, bucket, nil, nil, setTags(req.Tags)))
 	}
 	// Tagging names the new bucket, so a grant scoped to it applies.
 	if !allowed("s3vectors:CreateVectorBucket", "") ||
@@ -398,8 +402,8 @@ func (s *Server) handleListVectorBuckets(w http.ResponseWriter, r *http.Request)
 		}
 		arn := s.vectorBucketArn(b.Name)
 		pctx := s.policyContext(r, b.Name, tags, tags, tagRequest{})
-		if !s3tables.CheckPermissionWithContext("s3vectors:ListVectorBuckets", principal, owner, policy, arn, pctx) &&
-			!s3tables.CheckPermissionWithContext("s3vectors:GetVectorBucket", principal, owner, policy, arn, pctx) {
+		if !s.checkPermission(r, "s3vectors:ListVectorBuckets", principal, owner, policy, arn, pctx) &&
+			!s.checkPermission(r, "s3vectors:GetVectorBucket", principal, owner, policy, arn, pctx) {
 			continue
 		}
 		created, _ := time.Parse(time.RFC3339, b.CreatedAt)
@@ -923,7 +927,7 @@ func (s *Server) checkVectorPolicyDoc(r *http.Request, action, bucket string, re
 		return err
 	}
 	principal := s.tables.PrincipalFor(r)
-	if !s3tables.CheckPermissionWithContext(action, principal, record.OwnerAccountID,
+	if !s.checkPermission(r, action, principal, record.OwnerAccountID,
 		vectorPolicyFromExt(ext), s.vectorBucketArn(bucket), s.policyContext(r, bucket, tags, tags, tagRequest{})) {
 		return errVectorDenied
 	}
@@ -956,14 +960,14 @@ func (s *Server) authorizeVectorTags(ctx context.Context, r *http.Request, actio
 	resourceARN, resourceTags := s.vectorBucketArn(bucket), bucketTags
 	if index != "" {
 		resourceARN, resourceTags = s.indexArn(bucket, index), nil
-		if readsResourceTags(policy) {
+		if _, usesIAM := s.authenticator.(IAMAuthorizer); readsResourceTags(policy) || usesIAM {
 			if resourceTags, err = s.indexTags(ctx, bucket, index); err != nil {
 				return nil, err
 			}
 		}
 	}
 	principal := s.tables.PrincipalFor(r)
-	if !s3tables.CheckPermissionWithContext(action, principal, record.OwnerAccountID, policy, resourceARN,
+	if !s.checkPermission(r, action, principal, record.OwnerAccountID, policy, resourceARN,
 		s.policyContext(r, bucket, bucketTags, resourceTags, req)) {
 		return nil, &s3tables.S3TablesError{Type: s3tables.ErrCodeAccessDenied, Message: "access denied"}
 	}
@@ -1154,47 +1158,90 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) {
 		catalogErr(w, err)
 		return
 	}
-	// Only a caller who passed the create check may learn an index exists.
+	metadata := map[string]interface{}{
+		"dimension":      fmt.Sprint(req.Dimension),
+		"distanceMetric": strings.ToLower(req.DistanceMetric),
+		"createNonce":    uuid.NewString(),
+		"initializing":   "true",
+	}
+	var nonFilterable []string
+	if req.MetadataConfiguration != nil {
+		nonFilterable = slices.Clone(req.MetadataConfiguration.NonFilterableMetadataKeys)
+		keys, _ := json.Marshal(nonFilterable)
+		metadata["nonFilterableMetadataKeys"] = string(keys)
+	}
 	var existing map[string]interface{}
 	getErr := s.internalCatalog(r.Context(), "GetTable", &s3tables.GetTableRequest{
 		TableBucketARN: s.s3tablesArn(arns.bucket),
 		Namespace:      []string{indexNamespace},
 		Name:           arns.index,
 	}, &existing)
+	created := false
+	versionToken := ""
 	switch {
 	case getErr == nil:
-		writeError(w, http.StatusConflict, "ConflictException",
-			fmt.Sprintf("index %s already exists", arns.index))
-		return
-	case catalogErrCode(getErr) != s3tables.ErrCodeNoSuchTable:
-		catalogErr(w, getErr)
-		return
-	}
-	metadata := map[string]string{
-		"dimension":      fmt.Sprint(req.Dimension),
-		"distanceMetric": strings.ToLower(req.DistanceMetric),
-		// The nonce marks the record this request created, so a rollback after
-		// an engine failure can never erase a pre-existing index.
-		"createNonce": uuid.NewString(),
-	}
-	var nonFilterable []string
-	if req.MetadataConfiguration != nil {
-		nonFilterable = req.MetadataConfiguration.NonFilterableMetadataKeys
-		if keys, err := json.Marshal(nonFilterable); err == nil {
-			metadata["nonFilterableMetadataKeys"] = string(keys)
+		index := indexFromRecord(s, arns.bucket, arns.index, existing)
+		storedMetric, _ := distanceMetric(index.DistanceMetric)
+		var storedKeys []string
+		if index.MetadataConfiguration != nil {
+			storedKeys = slices.Clone(index.MetadataConfiguration.NonFilterableMetadataKeys)
 		}
-	}
-	metadataJSON, _ := json.Marshal(metadata)
-	err = s.internalCatalog(bound, "CreateTable", &s3tables.CreateTableRequest{
-		TableBucketARN: s.s3tablesArn(arns.bucket),
-		Namespace:      []string{indexNamespace},
-		Name:           arns.index,
-		Format:         s3tables.FormatLance,
-		Metadata:       &s3tables.TableMetadata{FullMetadata: metadataJSON},
-		Tags:           req.Tags,
-	}, nil)
-	if err != nil {
-		catalogErr(w, err)
+		slices.Sort(storedKeys)
+		slices.Sort(nonFilterable)
+		meta, _ := existing["metadata"].(map[string]interface{})
+		full, _ := meta["fullMetadata"].(map[string]interface{})
+		if full["initializing"] != "true" || index.Dimension != req.Dimension ||
+			storedMetric != metric ||
+			!slices.Equal(storedKeys, nonFilterable) {
+			writeError(w, http.StatusConflict, "ConflictException", fmt.Sprintf("index %s already exists", arns.index))
+			return
+		}
+		metadata = full
+		metadataJSON, _ := json.Marshal(metadata)
+		var response s3tables.UpdateTableResponse
+		if err := s.internalCatalog(bound, "UpdateTable", &s3tables.UpdateTableRequest{
+			TableBucketARN: s.s3tablesArn(arns.bucket),
+			Namespace:      []string{indexNamespace},
+			Name:           arns.index,
+			VersionToken:   recordVersionToken(existing),
+			Metadata:       &s3tables.TableMetadata{FullMetadata: metadataJSON},
+		}, &response); err != nil {
+			catalogErr(w, err)
+			return
+		}
+		versionToken = response.VersionToken
+	case catalogErrCode(getErr) == s3tables.ErrCodeNoSuchTable:
+		metadataJSON, _ := json.Marshal(metadata)
+		var response s3tables.CreateTableResponse
+		err = s.internalCatalog(bound, "CreateTable", &s3tables.CreateTableRequest{
+			TableBucketARN: s.s3tablesArn(arns.bucket),
+			Namespace:      []string{indexNamespace},
+			Name:           arns.index,
+			Format:         s3tables.FormatLance,
+			Metadata:       &s3tables.TableMetadata{FullMetadata: metadataJSON},
+			Tags:           req.Tags,
+		}, &response)
+		if err != nil {
+			catalogErr(w, err)
+			return
+		}
+		var record map[string]interface{}
+		if err := s.internalCatalog(bound, "GetTable", &s3tables.GetTableRequest{
+			TableBucketARN: s.s3tablesArn(arns.bucket),
+			Namespace:      []string{indexNamespace},
+			Name:           arns.index,
+		}, &record); err != nil {
+			catalogErr(w, err)
+			return
+		}
+		if recordNonce(record) != metadata["createNonce"] || recordVersionToken(record) != response.VersionToken {
+			writeError(w, http.StatusConflict, "ConflictException", "index changed during creation; retry the request")
+			return
+		}
+		created = true
+		versionToken = response.VersionToken
+	default:
+		catalogErr(w, getErr)
 		return
 	}
 	_, err = s.vectors.CreateIndex(r.Context(), &vector_pb.CreateIndexRequest{
@@ -1204,17 +1251,27 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) {
 		NonFilterableMetadataKeys: nonFilterable,
 	})
 	if err != nil {
-		// Roll the index record back so a half-created index does not linger.
-		// The delete runs past a client disconnect and only removes the record
-		// this request created: the nonce must match, and the versionToken
-		// compare-and-swap keeps a racing create from being erased.
+		if !created {
+			vectorErr(w, r, arns, err)
+			return
+		}
+		if !errors.Is(err, errNoVectorWorker) {
+			switch status.Code(err) {
+			case codes.InvalidArgument, codes.PermissionDenied, codes.Unauthenticated, codes.FailedPrecondition, codes.AlreadyExists, codes.NotFound, codes.OutOfRange, codes.Unimplemented:
+			default:
+				vectorErr(w, r, arns, err)
+				return
+			}
+		}
+		// Only definite failures roll back; an unknown outcome may have committed.
+		// The nonce and version token protect a concurrent replacement.
 		ctx := context.WithoutCancel(r.Context())
 		var rec map[string]interface{}
 		if gerr := s.internalCatalog(ctx, "GetTable", &s3tables.GetTableRequest{
 			TableBucketARN: s.s3tablesArn(arns.bucket),
 			Namespace:      []string{indexNamespace},
 			Name:           arns.index,
-		}, &rec); gerr == nil && recordNonce(rec) == metadata["createNonce"] {
+		}, &rec); gerr == nil && recordNonce(rec) == metadata["createNonce"] && recordVersionToken(rec) == versionToken {
 			var delResp map[string]interface{}
 			if derr := s.internalCatalog(ctx, "DeleteTable", &s3tables.DeleteTableRequest{
 				TableBucketARN: s.s3tablesArn(arns.bucket),
@@ -1230,6 +1287,18 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) {
 		vectorErr(w, r, arns, err)
 		return
 	}
+	delete(metadata, "initializing")
+	metadataJSON, _ := json.Marshal(metadata)
+	if err := s.internalCatalog(r.Context(), "UpdateTable", &s3tables.UpdateTableRequest{
+		TableBucketARN: s.s3tablesArn(arns.bucket),
+		Namespace:      []string{indexNamespace},
+		Name:           arns.index,
+		VersionToken:   versionToken,
+		Metadata:       &s3tables.TableMetadata{FullMetadata: metadataJSON},
+	}, nil); err != nil {
+		catalogErr(w, err)
+		return
+	}
 	writeJSON(w, http.StatusOK, &CreateIndexResponse{IndexArn: s.indexArn(arns.bucket, arns.index)})
 }
 
diff --git a/weed/s3api/s3vectors/iam.go b/weed/s3api/s3vectors/iam.go
new file mode 100644
index 0000000000..6ab9630d99
--- /dev/null
+++ b/weed/s3api/s3vectors/iam.go
@@ -0,0 +1,26 @@
+package s3vectors
+
+import (
+	"net/http"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+)
+
+type IAMAuthorizer interface {
+	EvaluateIAMPolicy(r *http.Request, action, resource string, policyContext *s3tables.PolicyContext) (allowed, denied bool)
+}
+
+func (s *Server) checkPermission(r *http.Request, action, principal, owner, policy, resource string, ctx *s3tables.PolicyContext) bool {
+	if authorizer, ok := s.authenticator.(IAMAuthorizer); ok {
+		allowed, denied := authorizer.EvaluateIAMPolicy(r, action, resource, ctx)
+		if denied {
+			return false
+		}
+		if allowed {
+			copyContext := *ctx
+			copyContext.IdentityActions = append(append([]string{}, ctx.IdentityActions...), action)
+			ctx = ©Context
+		}
+	}
+	return s3tables.CheckPermissionWithContext(action, principal, owner, policy, resource, ctx)
+}
diff --git a/weed/s3api/s3vectors/iam_test.go b/weed/s3api/s3vectors/iam_test.go
new file mode 100644
index 0000000000..41232675db
--- /dev/null
+++ b/weed/s3api/s3vectors/iam_test.go
@@ -0,0 +1,54 @@
+package s3vectors
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
+)
+
+type iamAuthenticator struct {
+	fakeAuthenticator
+	allowed, denied bool
+}
+
+func (a *iamAuthenticator) EvaluateIAMPolicy(*http.Request, string, string, *s3tables.PolicyContext) (bool, bool) {
+	return a.allowed, a.denied
+}
+
+func TestIAMGrantRespectsVectorResourceDenies(t *testing.T) {
+	auth := &iamAuthenticator{allowed: true}
+	server := &Server{authenticator: auth}
+	request := httptest.NewRequest("POST", "/QueryVectors", nil)
+	const action = "s3vectors:QueryVectors"
+	const resource = "arn:aws:s3vectors:us-east-1:000000000000:bucket/embeddings/index/docs"
+	ctx := &s3tables.PolicyContext{IdentityActions: []string{}}
+	if !server.checkPermission(request, action, "reader", "owner", "", resource, ctx) {
+		t.Fatal("IAM grant denied")
+	}
+	deny := `{"Statement":[{"Effect":"Deny","Principal":"*","Action":"s3vectors:*","Resource":"*"}]}`
+	if server.checkPermission(request, action, "reader", "owner", deny, resource, ctx) {
+		t.Fatal("resource deny was overridden by IAM")
+	}
+	auth.denied = true
+	if server.checkPermission(request, action, "owner", "owner", "", resource, ctx) {
+		t.Fatal("identity deny was overridden by ownership")
+	}
+}
+
+type requestTagAuthenticator struct{ fakeAuthenticator }
+
+func (*requestTagAuthenticator) EvaluateIAMPolicy(_ *http.Request, _, _ string, ctx *s3tables.PolicyContext) (bool, bool) {
+	return true, ctx.RequestTags["classification"] == "forbidden"
+}
+
+func TestIAMCreateBucketReceivesRequestTags(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	server := NewServer(s3tables.NewManagerClient(fs.Client), &requestTagAuthenticator{}, nil)
+	response := postJSONAs(t, "", server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "blocked", Tags: map[string]string{"classification": "forbidden"}})
+	if response.Code != http.StatusForbidden {
+		t.Fatalf("creation tag deny bypassed: %d %s", response.Code, response.Body)
+	}
+}
diff --git a/weed/s3api/s3vectors/operations.go b/weed/s3api/s3vectors/operations.go
new file mode 100644
index 0000000000..3f70b6b15a
--- /dev/null
+++ b/weed/s3api/s3vectors/operations.go
@@ -0,0 +1,12 @@
+package s3vectors
+
+import "sort"
+
+func Operations() []string {
+	operations := make([]string, 0, len(handlers))
+	for operation := range handlers {
+		operations = append(operations, operation)
+	}
+	sort.Strings(operations)
+	return operations
+}
diff --git a/weed/shell/commands.go b/weed/shell/commands.go
index 43b681d4d7..fa59f6039c 100644
--- a/weed/shell/commands.go
+++ b/weed/shell/commands.go
@@ -149,6 +149,10 @@ func (ce *CommandEnv) checkDirectory(path string) error {
 
 var _ = filer_pb.FilerClient(&CommandEnv{})
 
+func (ce *CommandEnv) GrpcDialOption() grpc.DialOption {
+	return ce.option.GrpcDialOption
+}
+
 func (ce *CommandEnv) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
 
 	return pb.WithGrpcFilerClient(streamingMode, 0, ce.FilerAddress(), ce.option.GrpcDialOption, fn)