vector: integrate engine health, admin management, and local deployment (#12517)

* vector: expose request metrics and registration-aware health endpoints

* vector: report immutable read cache and storage fetch metrics

* docs: describe vector engine health, metrics, and maintenance

* vector: count canceled requests when their handler is dropped

* vector: account for uncached large immutable storage reads

* vector: reliably release the stalled credential reader in tests

* vector: observe large reads as their streamed body is consumed

* vector: bound heartbeats on established master registration streams

* test: exercise hardened vector maintenance on Go and Rust volumes

* vector: register engines in their configured filer group

* vector: make retried index creation preserve compatible datasets

* vector: allow registration to advertise a separate address

* test: release stalled credential readers without blocking FIFO cleanup

* test: allow hardened vector coverage in a named filer group

* vector: discover engines in the same filer group as their gateway

* vector: fall back when the advertised address is empty

* admin: expose vector readiness, maintenance, and management tools (#12520)

* s3vectors: share vector discovery with trusted management clients

* admin: show vector engine readiness and recent maintenance activity

* admin: suggest S3 Vectors IAM actions from the supported operations

* admin: manage vector indexes, policies, tags, encryption, and query data

* admin: validate SSE-S3 support before creating encrypted vector buckets

* admin: discover vector engines in their registration group

* admin: probe vector engines concurrently with independent time budgets

* admin: verify encryption capability on the remote S3 gateway

* admin: defer discovery until after the HTTP server can start

* admin: distinguish unknown vector health services from failed engines

* admin: count workers that can execute vector index maintenance

* admin: allow read-only users to query vectors and inspect tags

* vector: scope gateway and admin discovery to the configured filer group

* admin: refresh policy state after mutations and serialize policy controls

* vector: drain active RPCs before closing replaced discovery connections

* vector: respect request cancellation while waiting for discovery

* vector: prefer another registered engine after a transport failure

* admin: check all masters before reporting no registered vector engines

* iam: honor attached vector policies without broadening legacy grants

* vector: allow short-lived discovery clients to close their connections

* vector: refresh cached master addresses for each discovery lookup

* iam: enforce vector bucket tag conditions in attached policies

* iam: authorize vector bucket creation with the requested tags

* test: exercise vector IAM grants and tag denies on Go and Rust volumes

* admin: reject authenticated vector mutations without an admin role

* test: run vector IAM assertions before the shared engine is stopped

* vector: add native mini startup, shell tools, and a runnable Compose example (#12522)

* s3vectors: share vector discovery with trusted management clients

* admin: show vector engine readiness and recent maintenance activity

* admin: suggest S3 Vectors IAM actions from the supported operations

* admin: manage vector indexes, policies, tags, encryption, and query data

* admin: validate SSE-S3 support before creating encrypted vector buckets

* admin: discover vector engines in their registration group

* admin: probe vector engines concurrently with independent time budgets

* admin: verify encryption capability on the remote S3 gateway

* admin: defer discovery until after the HTTP server can start

* admin: distinguish unknown vector health services from failed engines

* admin: count workers that can execute vector index maintenance

* admin: allow read-only users to query vectors and inspect tags

* vector: scope gateway and admin discovery to the configured filer group

* admin: refresh policy state after mutations and serialize policy controls

* vector: drain active RPCs before closing replaced discovery connections

* vector: respect request cancellation while waiting for discovery

* vector: prefer another registered engine after a transport failure

* admin: check all masters before reporting no registered vector engines

* iam: honor attached vector policies without broadening legacy grants

* vector: allow short-lived discovery clients to close their connections

* vector: refresh cached master addresses for each discovery lookup

* iam: enforce vector bucket tag conditions in attached policies

* iam: authorize vector bucket creation with the requested tags

* test: exercise vector IAM grants and tag denies on Go and Rust volumes

* admin: reject authenticated vector mutations without an admin role

* test: run vector IAM assertions before the shared engine is stopped

* mini: optionally start and supervise the vector engine

* mini: bootstrap vector buckets with existing Lance catalog helpers

* shell: manage vector buckets, indexes, and vector data

* docs: provide a runnable vector and maintenance worker Compose stack

* shell: scope vector discovery to the session and close its connections

* mini: honor the bind address for vector and metrics listeners

* mini: reserve vector ports before assigning gRPC and compatibility listeners

* mini: wait for vector storage setup and master registration

* mini: create requested vector buckets without requiring the Lance HTTP listener

* shell: refresh master addresses during vector discovery

* mini: resolve sibling vector executables with platform extensions

* vector: preserve uncertain creates and fix mini listener allocation (#12525)

* vector: retain catalog records when index creation has an unknown outcome

* mini: keep chosen listener ports reserved throughout allocation

* mini: resolve hostname binds before starting vector listeners

* vector: fix integration test imports and recover interrupted index creation (#12533)

* shell: register vector operations from the command package

* s3vectors: resume interrupted index initialization safely

* mini: exclude disabled WebDAV from listener allocation

* s3vectors: retain uncertain creates across refused retries

* s3vectors: claim pending indexes before retrying initialization

* s3vectors: compare resolved distance metrics during recovery

* s3vectors: preserve metadata keys when retrying initialization

* s3vectors: verify ownership before initializing new catalog records

.github/workflows/rust-vector-tests.yml

@@ -87,22 +87,17 @@ jobs:
         # volume, filer, gateway, engine) under mutual TLS, read JWTs
         # required, and every chunk encrypted by the gateway.
         hardened: ["", "1"]
-        worker: [""]
+        worker: ["", "1"]
         # Worker: the admin server and the Lance maintenance worker join the
         # cluster, for the tests that build vector indexes through a job.
-        include:
-          - volume: go
-            hardened: ""
-            worker: "1"
-          - volume: rust
-            hardened: ""
-            worker: "1"
+
     env:
       CARGO_PROFILE_DEV_DEBUG: "0"
       CARGO_BUILD_JOBS: "2"
       CARGO_TARGET_DIR: ${{ github.workspace }}/seaweed-vector/target
       VOLUME_SERVER_IMPL: ${{ matrix.volume }}
       VECTOR_QUERY_MODE: single
+      VECTOR_FILER_GROUP: vectors-ci
       VECTOR_TLS: ${{ matrix.hardened }}
       VECTOR_READ_JWT: ${{ matrix.hardened }}
       VECTOR_ENCRYPT_VOLUMES: ${{ matrix.hardened }}

docker/s3vectors/README.md

@@ -0,0 +1,39 @@
+# Vector buckets locally
+
+Run on Linux containers (amd64 or arm64):
+
+```sh
+docker compose up -d
+export AWS_ACCESS_KEY_ID=vector-demo
+export AWS_SECRET_ACCESS_KEY=vector-demo-secret
+export AWS_DEFAULT_REGION=us-east-1
+aws --endpoint-url http://localhost:8333 s3vectors create-index \
+  --vector-bucket-name embeddings --index-name images \
+  --data-type float32 --dimension 3 --distance-metric cosine
+aws --endpoint-url http://localhost:8333 s3vectors put-vectors \
+  --vector-bucket-name embeddings --index-name images \
+  --vectors '[{"key":"first","data":{"float32":[1,0,0]}}]'
+aws --endpoint-url http://localhost:8333 s3vectors query-vectors \
+  --vector-bucket-name embeddings --index-name images \
+  --query-vector '{"float32":[1,0,0]}' --top-k 1
+```
+
+The engine registers with the master. Wait for that registration before creating
+an index. The Lance worker builds and refreshes ANN indexes as data grows;
+queries can scan data before an ANN index is built. The demo's admin UI is at
+http://localhost:12535 (user `admin`, password `vector-demo`). Demo credentials
+and plaintext cluster connections are for this local example. Production
+settings are in the Helm/operator TLS and authentication documentation.
+
+`docker compose down` preserves the named data volume. Add `-v` to discard it.
+Override `SEAWEEDFS_IMAGE` to test a newer image. With metrics-enabled engines,
+append `--metrics-port=9329 --metrics-ip=0.0.0.0` to the vector command and scrape
+`vector:9329/metrics` inside the Compose network.
+
+A native development setup can use `weed mini -vector -vectorBucket=embeddings`
+with `weed-vector` beside `weed` or on PATH. `-vector.binary` selects an explicit
+binary, and `-vector.metricsPort=9329` enables HTTP readiness and metrics on a
+compatible engine. The child inherits the cluster's configuration directory and
+stops with mini. Run `weed-worker` separately for ANN maintenance. `weed shell`
+provides `s3vectors -operation <Operation> -file request.json` for the same API
+operations using trusted cluster credentials.

docker/s3vectors/compose.yaml

@@ -0,0 +1,34 @@
+services:
+  seaweedfs:
+    image: ${SEAWEEDFS_IMAGE:-chrislusf/seaweedfs:4.53}
+    command: mini -dir=/data -ip=seaweedfs -ip.bind=0.0.0.0 -admin.password=vector-demo -admin.worker.ip=0.0.0.0 -tableBucket=embeddings:LANCE
+    environment: &environment
+      WEED_JWT_FILER_SIGNING_KEY: vector-demo-cluster-key
+      AWS_ACCESS_KEY_ID: vector-demo
+      AWS_SECRET_ACCESS_KEY: vector-demo-secret
+    ports:
+      - "127.0.0.1:8333:8333"
+      - "127.0.0.1:12535:12535"
+    volumes:
+      - data:/data
+    healthcheck:
+      test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+  vector:
+    image: ${SEAWEEDFS_IMAGE:-chrislusf/seaweedfs:4.53}
+    command: vector --listen=vector:8090 --masters=seaweedfs:9333
+    environment: *environment
+    depends_on:
+      seaweedfs:
+        condition: service_healthy
+  worker:
+    image: ${SEAWEEDFS_IMAGE:-chrislusf/seaweedfs:4.53}
+    command: worker-rust --admin=seaweedfs:12535 --masters=seaweedfs:9333 --namespace=http://seaweedfs:9101
+    environment: *environment
+    depends_on:
+      seaweedfs:
+        condition: service_healthy
+volumes:
+  data:

seaweed-vector/Cargo.lock

@@ -6684,6 +6684,19 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "tonic-health"
+version = "0.14.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcfab99db777fba2802f0dfa861d1628d1ae916fb199d29819941f139ae85082"
+dependencies = [
+ "prost",
+ "tokio",
+ "tokio-stream",
+ "tonic",
+ "tonic-prost",
+]
+
 [[package]]
 name = "tonic-prost"
 version = "0.14.6"
@@ -7217,6 +7230,7 @@ dependencies = [
  "arrow-cast",
  "arrow-schema",
  "async-trait",
+ "axum",
  "bytes",
  "clap",
  "futures",
@@ -7225,6 +7239,7 @@ dependencies = [
  "lance-linalg",
  "moka",
  "object_store",
+ "prometheus",
  "prost",
  "prost-types",
  "protoc-bin-vendored",
@@ -7236,6 +7251,7 @@ dependencies = [
  "tokio",
  "tokio-stream",
  "tonic",
+ "tonic-health",
  "tonic-prost",
  "tonic-prost-build",
  "tracing",

seaweed-vector/README.md

@@ -0,0 +1,40 @@
+# SeaweedFS vector engine
+
+`weed-vector` serves the S3 Vectors data plane through the S3 gateway. Indexes use Lance tables on the cluster's filers and volume servers. Go and Rust volume servers use the same engine and API.
+
+With a local `weed mini` cluster running:
+
+```sh
+weed-vector --masters=localhost:9333 --metrics-port=9329
+```
+
+Point an AWS S3 Vectors client at the S3 endpoint, normally `http://localhost:8333`. The engine registers with the masters automatically. `--listen` defaults to `127.0.0.1:8090`; use an address reachable from the gateway when running on another host or in a container.
+
+| Setting | Default | Purpose |
+| --- | --- | --- |
+| `--masters` | Required | Comma-separated master addresses |
+| `--listen` | `127.0.0.1:8090` | gRPC listener and registered address |
+| `--filer-group` | Empty | Filer group used for table storage |
+| `--read-cache-mb` | `512` | Immutable data/index read cache; zero disables it |
+| `--nprobes` | Automatic | ANN partitions searched per query |
+| `--metrics-port` | `0` | HTTP metrics and health listener; zero disables it |
+| `--metrics-ip` | `127.0.0.1` | HTTP listener address |
+| `--security-toml` | Discovered | Cluster certificates, signing keys and encryption keys |
+
+The HTTP listener provides `/health`, `/ready`, and `/metrics`. Readiness requires an active registration with the master. The gRPC listener also implements the standard health service for `vector_pb.VectorService` and the whole server. These health statuses follow master registration.
+
+Metrics use the `SeaweedFS_vector_` prefix:
+
+- `connected`: active master registration.
+- `request_total{type,code}`, `request_seconds{type}`, and `in_flight_requests{type}`: engine requests and their gRPC outcomes.
+- `query_results_total`: vectors returned by queries.
+- `cache_requests_total{result}`, `cache_bytes`, and `cache_entries`: immutable read cache activity and occupancy.
+- `storage_read_bytes_total` and `storage_read_seconds`: storage fetches for cacheable immutable reads.
+
+The S3 gateway separately publishes `SeaweedFS_s3_` request metrics with `type="s3vectors.<Operation>"`. Scrape both processes to distinguish gateway latency from engine work. Metrics and HTTP probes are unauthenticated; bind them to an internal address or restrict their network access.
+
+For mutual TLS, configure `[grpc.vector]` for the engine's server certificate, `[grpc.client]` for its master/filer connections, and the cluster CA. HTTPS volume reads also use `[https.client]`. The engine follows the same signing and encryption configuration as the other cluster clients.
+
+Run the Lance maintenance worker against the same masters and Lance namespace to build and update ANN indexes. Queries can scan data that has not been indexed yet. See the [maintenance worker documentation](../seaweed-worker/README.md) and [Helm chart](../k8s/charts/seaweedfs/README.md) for deployment settings.
+
+The integration suite at `test/s3vectors` runs the same API and observability checks against both Go and Rust volume servers, including mutual TLS, signed volume reads, and encrypted volume data.

seaweed-vector/crates/server/Cargo.toml

@@ -13,6 +13,9 @@ name = "weed-vector"
 path = "src/main.rs"
 
 [dependencies]
+axum = "0.8"
+prometheus = { version = "0.13", default-features = false }
+tonic-health = "0.14"
 weed-vector-core = { path = "../core" }
 weed-lance-worker = { path = "../../../seaweed-worker/crates/lance" }
 sw-client = { path = "../../../seaweed-worker/crates/sw-client" }

seaweed-vector/crates/server/src/lib.rs

@@ -1,5 +1,7 @@
 //! The VectorService gRPC server the S3 gateway talks to.
 
+pub mod metrics;
+pub mod observed;
 pub mod read_cache;
 pub mod service;
 

seaweed-vector/crates/server/src/main.rs

@@ -5,12 +5,16 @@ use clap::Parser;
 use tokio_stream::wrappers::TcpListenerStream;
 use tonic::transport::Server;
 use weed_lance_worker::cluster::Cluster;
-use weed_vector_server::master_client;
 use weed_vector_server::pb::vector::vector_service_server::VectorServiceServer;
 use weed_vector_server::read_cache::ReadCache;
 use weed_vector_server::security::Security;
 use weed_vector_server::service::VectorServer;
 use weed_vector_server::tls::{self, ServerTls};
+use weed_vector_server::{
+    master_client,
+    metrics::{self, Metrics},
+    observed::ObservedService,
+};
 
 /// The VectorService the S3 gateway fans vector calls out to.
 #[derive(Parser)]
@@ -18,6 +22,9 @@ struct Args {
     /// gRPC listen address.
     #[arg(long, env = "WEED_VECTOR_LISTEN", default_value = "127.0.0.1:8090")]
     listen: String,
+    /// Address to register when it differs from the listen address.
+    #[arg(long, env = "WEED_VECTOR_ADVERTISE")]
+    advertise: Option<String>,
     /// Comma-separated master addresses. The engine registers with them, so
     /// the gateway finds it, and asks them for the filers it keeps tables
     /// through.
@@ -45,6 +52,21 @@ struct Args {
     /// processes find in -config_dir, ., ~/.seaweedfs and /etc/seaweedfs.
     #[arg(long, env = "WEED_VECTOR_SECURITY_TOML")]
     security_toml: Option<PathBuf>,
+    #[arg(
+        long,
+        alias = "metricsPort",
+        env = "WEED_VECTOR_METRICS_PORT",
+        default_value_t = 0
+    )]
+    metrics_port: u16,
+    #[arg(
+        long,
+        alias = "metricsIp",
+        env = "WEED_VECTOR_METRICS_IP",
+        default_value = "127.0.0.1"
+    )]
+    metrics_ip: String,
+
     /// The directory weed's -config_dir names, searched for security.toml
     /// first.
     #[arg(long, alias = "config_dir", env = "WEED_VECTOR_CONFIG_DIR")]
@@ -78,7 +100,13 @@ fn normalize_args(args: Vec<String>) -> Vec<String> {
 
 /// A wildcard listen address is not reachable for peers; derive the outbound
 /// IP toward the first master instead — the address the master sees anyway.
-fn advertised_addr(listen: &str, masters: &[String]) -> String {
+fn advertised_addr(listen: &str, masters: &[String], advertise: Option<&str>) -> String {
+    if let Some(address) = advertise
+        .map(str::trim)
+        .filter(|address| !address.is_empty())
+    {
+        return address.to_string();
+    }
     let Some((host, _)) = listen.rsplit_once(':') else {
         return listen.to_string();
     };
@@ -128,11 +156,31 @@ async fn main() -> Result<()> {
         .with_context(|| format!("listen on {}", args.listen))?;
     let cluster = Cluster::new(&args.masters, &args.filer_group, &security)?;
     let reads = cluster.storage(&security).await?;
+    let (health, health_service) = tonic_health::server::health_reporter();
+    let metrics = Metrics::new()?.with_health(health);
+    metrics.set_connected(false).await;
+    let cache = (args.read_cache_mb > 0)
+        .then(|| ReadCache::new(args.read_cache_mb << 20).with_metrics(metrics.clone()));
     let mut service = VectorServer::new()
         .with_nprobes(args.nprobes.map(std::num::NonZeroUsize::get))
         .with_storage(reads);
-    if args.read_cache_mb > 0 {
-        service = service.with_read_cache(ReadCache::new(args.read_cache_mb << 20));
+    if let Some(cache) = &cache {
+        service = service.with_read_cache(cache.clone());
+    }
+    if args.metrics_port > 0 {
+        let ip = args
+            .metrics_ip
+            .trim_start_matches('[')
+            .trim_end_matches(']')
+            .parse::<std::net::IpAddr>()?;
+        let listener =
+            tokio::net::TcpListener::bind(std::net::SocketAddr::new(ip, args.metrics_port)).await?;
+        let metrics = metrics.clone();
+        tokio::spawn(async move {
+            if let Err(error) = metrics::serve_listener(metrics, listener, cache).await {
+                tracing::error!(%error, "vector metrics server stopped");
+            }
+        });
     }
     tracing::info!(
         listen = %args.listen,
@@ -148,19 +196,25 @@ async fn main() -> Result<()> {
     );
     // Registered only now, with storage connected and the listener bound,
     // so the gateway never finds an engine that cannot serve yet.
-    let advertise = advertised_addr(&args.listen, &cluster.masters);
+    let advertise = advertised_addr(&args.listen, &cluster.masters, args.advertise.as_deref());
     tokio::spawn(master_client::keep_connected(
         cluster.masters.clone(),
         advertise,
+        args.filer_group.clone(),
         cluster.master_tls.clone(),
+        metrics.clone(),
     ));
     let router = Server::builder()
+        .add_service(health_service)
         // Match the gateway's 1 GiB cap — put batches over the tonic 4 MiB
         // default otherwise get rejected before they reach the handler.
         .add_service(
-            VectorServiceServer::new(service)
-                .max_decoding_message_size(1 << 30)
-                .max_encoding_message_size(1 << 30),
+            VectorServiceServer::new(ObservedService {
+                inner: service,
+                metrics,
+            })
+            .max_decoding_message_size(1 << 30)
+            .max_encoding_message_size(1 << 30),
         );
     match server_tls {
         Some(tls) => {
@@ -180,6 +234,20 @@ async fn main() -> Result<()> {
 mod tests {
     use super::*;
 
+    #[test]
+    fn an_empty_advertise_uses_the_listen_address() {
+        for value in [None, Some(""), Some("  ")] {
+            assert_eq!(
+                advertised_addr("127.0.0.1:8090", &[], value),
+                "127.0.0.1:8090"
+            );
+        }
+        assert_eq!(
+            advertised_addr("127.0.0.1:8090", &[], Some("vector.example:8090")),
+            "vector.example:8090"
+        );
+    }
+
     #[test]
     fn nprobes_must_be_positive() {
         let parse = |nprobes: &str| {

seaweed-vector/crates/server/src/master_client.rs

@@ -8,16 +8,18 @@ use std::time::Duration;
 use anyhow::{Context, Result};
 use tokio::time::sleep;
 
-use crate::master_pb::{KeepConnectedRequest, seaweed_client::SeaweedClient};
+use crate::master_pb::{
+    KeepConnectedRequest, ListClusterNodesRequest, seaweed_client::SeaweedClient,
+};
 use crate::security::GrpcTls;
 pub use weed_lance_worker::cluster::{CONNECT_TIMEOUT, connect, grpc_addr};
 
-fn registration(client_addr: &str) -> KeepConnectedRequest {
+fn registration(client_addr: &str, filer_group: &str) -> KeepConnectedRequest {
     KeepConnectedRequest {
         client_type: "vector".into(),
         client_address: client_addr.into(),
         version: env!("CARGO_PKG_VERSION").into(),
-        filer_group: String::new(),
+        filer_group: filer_group.into(),
         data_center: String::new(),
         rack: String::new(),
         leave_lock_ring: false,
@@ -39,12 +41,22 @@ fn http_addr(master: &str) -> String {
 
 /// Retry forever: try each master until a stream opens, hold it until it
 /// errors, then rediscover — same shape as wdclient.KeepConnectedToMaster.
-pub async fn keep_connected(masters: Vec<String>, client_addr: String, tls: Option<GrpcTls>) {
+pub async fn keep_connected(
+    masters: Vec<String>,
+    client_addr: String,
+    filer_group: String,
+    tls: Option<GrpcTls>,
+    metrics: crate::metrics::Metrics,
+) {
     let client_addr = client_addr.as_str();
+    let filer_group = filer_group.as_str();
     let tls = tls.as_ref();
+    let metrics = &metrics;
     loop {
         try_all_masters(&masters, |master| async move {
-            stream_to(&master, client_addr, tls).await
+            let result = stream_to(&master, client_addr, filer_group, tls, Some(&metrics)).await;
+            metrics.set_connected(false).await;
+            result
         })
         .await;
         sleep(Duration::from_secs(3)).await;
@@ -83,14 +95,16 @@ where
 async fn stream_to(
     master: &str,
     client_addr: &str,
+    filer_group: &str,
     tls: Option<&GrpcTls>,
+    metrics: Option<&crate::metrics::Metrics>,
 ) -> Result<Option<String>> {
     let channel = connect(master, tls).await?;
     // A master answers at once, a follower with its leader hint.
     let answer_by = tokio::time::Instant::now() + CONNECT_TIMEOUT;
     let mut client = SeaweedClient::new(channel);
     let (tx, rx) = tokio::sync::mpsc::channel(8);
-    tx.send(registration(client_addr)).await?;
+    tx.send(registration(client_addr, filer_group)).await?;
     let mut stream = tokio::time::timeout_at(
         answer_by,
         client.keep_connected(tokio_stream::wrappers::ReceiverStream::new(rx)),
@@ -99,6 +113,8 @@ async fn stream_to(
     .context("master did not answer keep-connected")??
     .into_inner();
     let mut answered = false;
+    let interval = Duration::from_secs(5);
+    let mut heartbeat = tokio::time::interval_at(tokio::time::Instant::now() + interval, interval);
     // The master keeps the node live while the stream is open; resend the
     // registration periodically so a reconnect never leaves a stale entry.
     loop {
@@ -117,23 +133,66 @@ async fn stream_to(
                             return Ok(Some(loc.leader));
                         }
                     }
+                    if let Some(metrics) = metrics { metrics.set_connected(true).await; }
                 }
                 None => anyhow::bail!("master closed the keep-connected stream"),
             },
             _ = tokio::time::sleep_until(answer_by), if !answered => {
                 anyhow::bail!("master did not answer keep-connected");
             }
-            _ = sleep(Duration::from_secs(5)) => {
-                tx.send(registration(client_addr)).await?;
+            _ = heartbeat.tick() => {
+                tokio::time::timeout(CONNECT_TIMEOUT, tx.send(registration(client_addr, filer_group))).await
+                    .context("registration send stalled")??;
+                probe_registration(&mut client).await?;
             }
         }
     }
 }
 
+async fn probe_registration(client: &mut SeaweedClient<tonic::transport::Channel>) -> Result<()> {
+    tokio::time::timeout(
+        CONNECT_TIMEOUT,
+        client.list_cluster_nodes(ListClusterNodesRequest {
+            client_type: "vector".into(),
+            ..Default::default()
+        }),
+    )
+    .await
+    .context("registration heartbeat stalled")??;
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[test]
+    fn registration_preserves_filer_group() {
+        assert_eq!(
+            registration("vector:8090", "tenant-a").filer_group,
+            "tenant-a"
+        );
+    }
+
+    #[tokio::test]
+    async fn stalled_registration_heartbeat_is_bounded() {
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let address = listener.local_addr().unwrap();
+        let hold = tokio::spawn(async move {
+            let (_connection, _) = listener.accept().await.unwrap();
+            std::future::pending::<()>().await;
+        });
+        let channel = tonic::transport::Endpoint::from_shared(format!("http://{address}"))
+            .unwrap()
+            .connect_lazy();
+        let mut client = SeaweedClient::new(channel);
+        let result = tokio::time::timeout(CONNECT_TIMEOUT * 3, probe_registration(&mut client))
+            .await
+            .unwrap();
+        assert!(format!("{:#}", result.unwrap_err()).contains("registration heartbeat stalled"));
+        hold.abort();
+    }
+
     /// Runs one pass against scripted masters and returns the dial order.
     async fn pass(masters: &[&str], answer: fn(&str) -> Result<Option<String>>) -> Vec<String> {
         let masters: Vec<String> = masters.iter().map(|m| m.to_string()).collect();
@@ -183,7 +242,7 @@ mod tests {
     #[tokio::test]
     async fn dial_to_a_black_holed_master_is_bounded() {
         // TEST-NET-1 (RFC 5737) is reserved, so nothing answers the SYN.
-        let dial = stream_to("192.0.2.1:9333", "127.0.0.1:8080", None);
+        let dial = stream_to("192.0.2.1:9333", "127.0.0.1:8080", "", None, None);
         let result = tokio::time::timeout(CONNECT_TIMEOUT * 3, dial).await;
         assert!(matches!(result, Ok(Err(_))), "dial was not bounded");
     }
@@ -197,7 +256,7 @@ mod tests {
             let _ = tokio::io::copy(&mut conn, &mut tokio::io::sink()).await;
         });
         let master = format!("127.0.0.1:9333.{port}");
-        let open = stream_to(&master, "127.0.0.1:8080", None);
+        let open = stream_to(&master, "127.0.0.1:8080", "", None, None);
         match tokio::time::timeout(CONNECT_TIMEOUT * 3, open).await {
             Ok(Err(e)) => assert!(format!("{e:#}").contains("did not answer"), "{e:#}"),
             other => panic!("open was not bounded: {other:?}"),
@@ -232,7 +291,7 @@ mod tests {
         });
         let master = format!("127.0.0.1:9333.{port}");
         let tls = credentials();
-        let open = stream_to(&master, "127.0.0.1:8080", Some(&tls));
+        let open = stream_to(&master, "127.0.0.1:8080", "", Some(&tls), None);
         match tokio::time::timeout(CONNECT_TIMEOUT * 3, open).await {
             Ok(Err(e)) => assert!(format!("{e:#}").contains("did not complete"), "{e:#}"),
             other => panic!("handshake was not bounded: {other:?}"),
@@ -258,16 +317,19 @@ mod tests {
         let mut tls = credentials();
         tls.ca_path = fifo.display().to_string();
         let master = format!("127.0.0.1:9333.{port}");
-        let open = stream_to(&master, "127.0.0.1:8080", Some(&tls));
+        let open = stream_to(&master, "127.0.0.1:8080", "", Some(&tls), None);
         let result = tokio::time::timeout(CONNECT_TIMEOUT * 3, open).await;
-        // A read+write open of a FIFO does not block, and it lets the
-        // blocked read finish so the runtime can shut down.
-        drop(
-            std::fs::OpenOptions::new()
-                .read(true)
-                .write(true)
-                .open(&fifo),
-        );
+        use std::io::Write;
+        let mut cleanup = std::fs::OpenOptions::new()
+            .read(true)
+            .write(true)
+            .open(&fifo)
+            .unwrap();
+        let replacement = dir.path().join("ca.replacement");
+        std::fs::write(&replacement, b"invalid CA").unwrap();
+        std::fs::rename(&replacement, &fifo).unwrap();
+        cleanup.write_all(b"invalid CA").unwrap();
+        drop(cleanup);
         match result {
             Ok(Err(e)) => assert!(format!("{e:#}").contains("did not complete"), "{e:#}"),
             other => panic!("credential read was not bounded: {other:?}"),

seaweed-vector/crates/server/src/metrics.rs

@@ -0,0 +1,341 @@
+use std::net::SocketAddr;
+
+use anyhow::Result;
+use axum::{Router, http::StatusCode, routing::get};
+use prometheus::{
+    Encoder, Histogram, HistogramOpts, HistogramVec, IntCounter, IntCounterVec, IntGauge,
+    IntGaugeVec, Opts, Registry, TextEncoder,
+};
+use tonic_health::{ServingStatus, server::HealthReporter};
+
+use crate::read_cache::ReadCache;
+
+#[derive(Clone)]
+pub struct Metrics {
+    pub registry: Registry,
+    pub connected: IntGauge,
+    pub requests: IntCounterVec,
+    pub request_seconds: HistogramVec,
+    pub in_flight: IntGaugeVec,
+    pub query_results: IntCounter,
+    pub cache_requests: IntCounterVec,
+    pub storage_bytes: IntCounter,
+    pub storage_read_seconds: Histogram,
+    cache_bytes: IntGauge,
+    cache_entries: IntGauge,
+    health: Option<HealthReporter>,
+}
+
+impl Metrics {
+    pub fn new() -> Result<Self> {
+        let registry = Registry::new();
+        let opts = |name: &str, help: &str| {
+            Opts::new(name, help)
+                .namespace("SeaweedFS")
+                .subsystem("vector")
+        };
+        let buckets = prometheus::exponential_buckets(0.0001, 2.0, 24)?;
+        let histogram =
+            |name: &str, help: &str| HistogramOpts::from(opts(name, help)).buckets(buckets.clone());
+        let connected = IntGauge::with_opts(opts(
+            "connected",
+            "Whether the engine is registered with the master.",
+        ))?;
+        let requests = IntCounterVec::new(
+            opts(
+                "request_total",
+                "Vector engine requests by operation and gRPC status.",
+            ),
+            &["type", "code"],
+        )?;
+        let request_seconds = HistogramVec::new(
+            histogram("request_seconds", "Vector engine request duration."),
+            &["type"],
+        )?;
+        let in_flight = IntGaugeVec::new(
+            opts("in_flight_requests", "Vector engine requests in progress."),
+            &["type"],
+        )?;
+        let query_results =
+            IntCounter::with_opts(opts("query_results_total", "Vectors returned by queries."))?;
+        let cache_requests = IntCounterVec::new(
+            opts("cache_requests_total", "Immutable reads by cache outcome."),
+            &["result"],
+        )?;
+        let storage_bytes = IntCounter::with_opts(opts(
+            "storage_read_bytes_total",
+            "Bytes fetched for immutable cache reads.",
+        ))?;
+        let storage_read_seconds = Histogram::with_opts(histogram(
+            "storage_read_seconds",
+            "Immutable storage fetch duration.",
+        ))?;
+        let cache_bytes =
+            IntGauge::with_opts(opts("cache_bytes", "Current cache weight in bytes."))?;
+        let cache_entries = IntGauge::with_opts(opts("cache_entries", "Current cache entries."))?;
+        for collector in [
+            Box::new(connected.clone()) as Box<dyn prometheus::core::Collector>,
+            Box::new(requests.clone()),
+            Box::new(request_seconds.clone()),
+            Box::new(in_flight.clone()),
+            Box::new(query_results.clone()),
+            Box::new(cache_requests.clone()),
+            Box::new(storage_bytes.clone()),
+            Box::new(storage_read_seconds.clone()),
+            Box::new(cache_bytes.clone()),
+            Box::new(cache_entries.clone()),
+        ] {
+            registry.register(collector)?;
+        }
+        Ok(Self {
+            registry,
+            connected,
+            requests,
+            request_seconds,
+            in_flight,
+            query_results,
+            cache_requests,
+            storage_bytes,
+            storage_read_seconds,
+            cache_bytes,
+            cache_entries,
+            health: None,
+        })
+    }
+
+    pub fn with_health(mut self, health: HealthReporter) -> Self {
+        self.health = Some(health);
+        self
+    }
+
+    pub async fn set_connected(&self, connected: bool) {
+        self.connected.set(i64::from(connected));
+        if let Some(health) = self.health.clone() {
+            let status = if connected {
+                ServingStatus::Serving
+            } else {
+                ServingStatus::NotServing
+            };
+            health.set_service_status("", status).await;
+            health
+                .set_service_status("vector_pb.VectorService", status)
+                .await;
+        }
+    }
+
+    pub fn start(&self, operation: &str) -> InFlight {
+        let gauge = self.in_flight.with_label_values(&[operation]);
+        gauge.inc();
+        InFlight {
+            gauge,
+            requests: self.requests.clone(),
+            operation: operation.into(),
+            completed: false,
+            _timer: self
+                .request_seconds
+                .with_label_values(&[operation])
+                .start_timer(),
+        }
+    }
+
+    fn gather(&self, cache: Option<&ReadCache>) -> Result<String> {
+        if let Some(cache) = cache {
+            let (entries, bytes) = cache.size();
+            self.cache_entries.set(entries as i64);
+            self.cache_bytes.set(bytes as i64);
+        }
+        let mut buffer = Vec::new();
+        TextEncoder::new().encode(&self.registry.gather(), &mut buffer)?;
+        Ok(String::from_utf8(buffer)?)
+    }
+}
+
+pub struct InFlight {
+    gauge: IntGauge,
+    requests: IntCounterVec,
+    operation: String,
+    completed: bool,
+    _timer: prometheus::HistogramTimer,
+}
+
+impl InFlight {
+    pub fn complete(&mut self, error: Option<tonic::Code>) {
+        let code = error
+            .map(|c| format!("{c:?}"))
+            .unwrap_or_else(|| "OK".into());
+        self.requests
+            .with_label_values(&[&self.operation, &code])
+            .inc();
+        self.completed = true;
+    }
+}
+
+impl Drop for InFlight {
+    fn drop(&mut self) {
+        self.gauge.dec();
+        if !self.completed {
+            self.requests
+                .with_label_values(&[&self.operation, "Cancelled"])
+                .inc();
+        }
+    }
+}
+
+pub async fn serve(metrics: Metrics, address: SocketAddr, cache: Option<ReadCache>) -> Result<()> {
+    let listener = tokio::net::TcpListener::bind(address).await?;
+    serve_listener(metrics, listener, cache).await
+}
+
+pub async fn serve_listener(
+    metrics: Metrics,
+    listener: tokio::net::TcpListener,
+    cache: Option<ReadCache>,
+) -> Result<()> {
+    let readiness = metrics.clone();
+    let app = Router::new()
+        .route("/health", get(|| async { StatusCode::OK }))
+        .route(
+            "/ready",
+            get(move || {
+                let metrics = readiness.clone();
+                async move {
+                    if metrics.connected.get() == 1 {
+                        StatusCode::OK
+                    } else {
+                        StatusCode::SERVICE_UNAVAILABLE
+                    }
+                }
+            }),
+        )
+        .route(
+            "/metrics",
+            get(move || {
+                let metrics = metrics.clone();
+                let cache = cache.clone();
+                async move {
+                    metrics
+                        .gather(cache.as_ref())
+                        .map(|body| ([("content-type", "text/plain; version=0.0.4")], body))
+                        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)
+                }
+            }),
+        );
+    axum::serve(listener, app).await?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn endpoints_follow_registration_and_publish_metrics() {
+        let metrics = Metrics::new().unwrap();
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let endpoint = format!("http://{}", listener.local_addr().unwrap());
+        let task = tokio::spawn(serve_listener(metrics.clone(), listener, None));
+        let client = reqwest::Client::new();
+        assert_eq!(
+            client
+                .get(format!("{endpoint}/health"))
+                .send()
+                .await
+                .unwrap()
+                .status(),
+            200
+        );
+        assert_eq!(
+            client
+                .get(format!("{endpoint}/ready"))
+                .send()
+                .await
+                .unwrap()
+                .status(),
+            503
+        );
+        metrics.set_connected(true).await;
+        assert_eq!(
+            client
+                .get(format!("{endpoint}/ready"))
+                .send()
+                .await
+                .unwrap()
+                .status(),
+            200
+        );
+        let in_flight = metrics.start("QueryVectors");
+        let body = client
+            .get(format!("{endpoint}/metrics"))
+            .send()
+            .await
+            .unwrap()
+            .text()
+            .await
+            .unwrap();
+        assert!(body.contains("SeaweedFS_vector_connected 1"));
+        assert!(body.contains("SeaweedFS_vector_in_flight_requests{type=\"QueryVectors\"} 1"));
+        drop(in_flight);
+        assert_eq!(
+            metrics.in_flight.with_label_values(&["QueryVectors"]).get(),
+            0
+        );
+        metrics.set_connected(false).await;
+        assert_eq!(
+            client
+                .get(format!("{endpoint}/ready"))
+                .send()
+                .await
+                .unwrap()
+                .status(),
+            503
+        );
+        task.abort();
+    }
+
+    #[tokio::test]
+    async fn grpc_health_follows_registration() {
+        use tonic_health::pb::{HealthCheckRequest, health_client::HealthClient};
+        let (reporter, service) = tonic_health::server::health_reporter();
+        let metrics = Metrics::new().unwrap().with_health(reporter);
+        metrics.set_connected(false).await;
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let endpoint = format!("http://{}", listener.local_addr().unwrap());
+        let task = tokio::spawn(
+            tonic::transport::Server::builder()
+                .add_service(service)
+                .serve_with_incoming(tokio_stream::wrappers::TcpListenerStream::new(listener)),
+        );
+        let channel = tonic::transport::Endpoint::from_shared(endpoint)
+            .unwrap()
+            .connect()
+            .await
+            .unwrap();
+        let mut client = HealthClient::new(channel);
+        let request = || HealthCheckRequest {
+            service: "vector_pb.VectorService".into(),
+        };
+        assert_eq!(
+            client.check(request()).await.unwrap().into_inner().status,
+            2
+        );
+        metrics.set_connected(true).await;
+        assert_eq!(
+            client.check(request()).await.unwrap().into_inner().status,
+            1
+        );
+        metrics.set_connected(false).await;
+        assert_eq!(
+            client.check(request()).await.unwrap().into_inner().status,
+            2
+        );
+        task.abort();
+    }
+}
+
+impl std::fmt::Debug for Metrics {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Metrics")
+            .field("connected", &self.connected.get())
+            .finish()
+    }
+}

seaweed-vector/crates/server/src/observed.rs

@@ -0,0 +1,122 @@
+use crate::metrics::Metrics;
+use crate::pb::vector::{self as pb, vector_service_server::VectorService};
+use crate::service::VectorServer;
+use tonic::{Request, Response, Status};
+
+pub struct ObservedService {
+    pub inner: VectorServer,
+    pub metrics: Metrics,
+}
+
+#[tonic::async_trait]
+impl VectorService for ObservedService {
+    async fn create_index(
+        &self,
+        request: Request<pb::CreateIndexRequest>,
+    ) -> Result<Response<pb::CreateIndexResponse>, Status> {
+        let mut in_flight = self.metrics.start("CreateIndex");
+        let result = self.inner.create_index(request).await;
+        in_flight.complete(result.as_ref().err().map(|e| e.code()));
+        result
+    }
+    async fn drop_index(
+        &self,
+        request: Request<pb::DropIndexRequest>,
+    ) -> Result<Response<pb::DropIndexResponse>, Status> {
+        let mut in_flight = self.metrics.start("DropIndex");
+        let result = self.inner.drop_index(request).await;
+        in_flight.complete(result.as_ref().err().map(|e| e.code()));
+        result
+    }
+    async fn put_vectors(
+        &self,
+        request: Request<pb::PutVectorsRequest>,
+    ) -> Result<Response<pb::PutVectorsResponse>, Status> {
+        let mut in_flight = self.metrics.start("PutVectors");
+        let result = self.inner.put_vectors(request).await;
+        in_flight.complete(result.as_ref().err().map(|e| e.code()));
+        result
+    }
+    async fn get_vectors(
+        &self,
+        request: Request<pb::GetVectorsRequest>,
+    ) -> Result<Response<pb::GetVectorsResponse>, Status> {
+        let mut in_flight = self.metrics.start("GetVectors");
+        let result = self.inner.get_vectors(request).await;
+        in_flight.complete(result.as_ref().err().map(|e| e.code()));
+        result
+    }
+    async fn delete_vectors(
+        &self,
+        request: Request<pb::DeleteVectorsRequest>,
+    ) -> Result<Response<pb::DeleteVectorsResponse>, Status> {
+        let mut in_flight = self.metrics.start("DeleteVectors");
+        let result = self.inner.delete_vectors(request).await;
+        in_flight.complete(result.as_ref().err().map(|e| e.code()));
+        result
+    }
+    async fn list_vectors(
+        &self,
+        request: Request<pb::ListVectorsRequest>,
+    ) -> Result<Response<pb::ListVectorsResponse>, Status> {
+        let mut in_flight = self.metrics.start("ListVectors");
+        let result = self.inner.list_vectors(request).await;
+        in_flight.complete(result.as_ref().err().map(|e| e.code()));
+        result
+    }
+    async fn query_vectors(
+        &self,
+        request: Request<pb::QueryVectorsRequest>,
+    ) -> Result<Response<pb::QueryVectorsResponse>, Status> {
+        let mut in_flight = self.metrics.start("QueryVectors");
+        let result = self.inner.query_vectors(request).await;
+        in_flight.complete(result.as_ref().err().map(|e| e.code()));
+        if let Ok(response) = &result {
+            self.metrics
+                .query_results
+                .inc_by(response.get_ref().vectors.len() as u64);
+        }
+        result
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn failures_are_counted_and_release_in_flight_requests() {
+        let service = ObservedService {
+            inner: VectorServer::new(),
+            metrics: Metrics::new().unwrap(),
+        };
+        let response = service
+            .query_vectors(Request::new(pb::QueryVectorsRequest::default()))
+            .await;
+        let code = format!("{:?}", response.unwrap_err().code());
+        assert_eq!(
+            service
+                .metrics
+                .requests
+                .with_label_values(&["QueryVectors", &code])
+                .get(),
+            1
+        );
+        assert_eq!(
+            service
+                .metrics
+                .in_flight
+                .with_label_values(&["QueryVectors"])
+                .get(),
+            0
+        );
+        assert_eq!(
+            service
+                .metrics
+                .request_seconds
+                .with_label_values(&["QueryVectors"])
+                .get_sample_count(),
+            1
+        );
+    }
+}

seaweed-vector/crates/server/src/read_cache.rs

@@ -9,6 +9,7 @@ use std::fmt;
 use std::ops::Range;
 use std::sync::Arc;
 
+use crate::metrics::Metrics;
 use bytes::Bytes;
 use futures::StreamExt;
 use futures::stream::BoxStream;
@@ -42,6 +43,7 @@ pub struct ReadCache {
     /// Keeps entries apart from those of another [`Self::scoped`] view, for
     /// stores that share a prefix but not their bytes.
     scope: Arc<str>,
+    metrics: Option<Metrics>,
 }
 
 impl ReadCache {
@@ -54,14 +56,26 @@ impl ReadCache {
                 })
                 .build(),
             scope: "".into(),
+            metrics: None,
         }
     }
 
+    pub fn with_metrics(mut self, metrics: Metrics) -> Self {
+        self.metrics = Some(metrics);
+        self
+    }
+
+    pub fn size(&self) -> (u64, u64) {
+        self.cache.run_pending_tasks();
+        (self.cache.entry_count(), self.cache.weighted_size())
+    }
+
     /// The same cache, keyed apart under `scope`.
     pub fn scoped(&self, scope: &str) -> Self {
         Self {
             cache: self.cache.clone(),
             scope: scope.into(),
+            metrics: self.metrics.clone(),
         }
     }
 }
@@ -81,6 +95,7 @@ impl WrappingObjectStore for ReadCache {
             inner: original,
             store: format!("{}|{store_prefix}", self.scope).into(),
             cache: self.cache.clone(),
+            metrics: self.metrics.clone(),
         })
     }
 }
@@ -136,6 +151,7 @@ struct CachedStore {
     inner: Arc<dyn ObjectStore>,
     store: Arc<str>,
     cache: moka::sync::Cache<Key, Entry>,
+    metrics: Option<Metrics>,
 }
 
 impl CachedStore {
@@ -184,14 +200,47 @@ impl ObjectStore for CachedStore {
             object: Some(object),
         }) = self.cache.get(&key)
         {
+            if let Some(metrics) = &self.metrics {
+                metrics.cache_requests.with_label_values(&["hit"]).inc();
+            }
             return Ok(whole(bytes, &object));
         }
+        if let Some(metrics) = &self.metrics {
+            metrics.cache_requests.with_label_values(&["miss"]).inc();
+        }
+        let _timer = self
+            .metrics
+            .as_ref()
+            .map(|m| m.storage_read_seconds.start_timer());
         let result = self.inner.get_opts(location, options).await?;
         if result.meta.size > MAX_WHOLE_OBJECT {
-            return Ok(result);
+            let meta = result.meta.clone();
+            let range = result.range.clone();
+            let attributes = result.attributes.clone();
+            let metrics = self.metrics.clone();
+            let timer = _timer;
+            let stream = result
+                .into_stream()
+                .map(move |chunk| {
+                    let _timer = &timer;
+                    if let (Some(metrics), Ok(bytes)) = (&metrics, &chunk) {
+                        metrics.storage_bytes.inc_by(bytes.len() as u64);
+                    }
+                    chunk
+                })
+                .boxed();
+            return Ok(GetResult {
+                payload: GetResultPayload::Stream(stream),
+                meta,
+                range,
+                attributes,
+            });
         }
         let object = Arc::new((result.meta.clone(), result.attributes.clone()));
         let bytes = owned(result.bytes().await?);
+        if let Some(metrics) = &self.metrics {
+            metrics.storage_bytes.inc_by(bytes.len() as u64);
+        }
         self.cache.insert(
             key,
             Entry {
@@ -220,7 +269,21 @@ impl ObjectStore for CachedStore {
             .filter(|(_, hit)| hit.is_none())
             .map(|(range, _)| range.clone())
             .collect();
+        if let Some(metrics) = &self.metrics {
+            metrics
+                .cache_requests
+                .with_label_values(&["miss"])
+                .inc_by(missing.len() as u64);
+            metrics
+                .cache_requests
+                .with_label_values(&["hit"])
+                .inc_by((ranges.len() - missing.len()) as u64);
+        }
         if !missing.is_empty() {
+            let _timer = self
+                .metrics
+                .as_ref()
+                .map(|m| m.storage_read_seconds.start_timer());
             let mut fetched = self.inner.get_ranges(location, &missing).await?.into_iter();
             for (range, slot) in ranges.iter().zip(out.iter_mut()) {
                 if slot.is_some() {
@@ -230,6 +293,9 @@ impl ObjectStore for CachedStore {
                     break;
                 };
                 let bytes = owned(bytes);
+                if let Some(metrics) = &self.metrics {
+                    metrics.storage_bytes.inc_by(bytes.len() as u64);
+                }
                 self.cache.insert(
                     self.key(location, Some(range.clone())),
                     Entry {
@@ -491,6 +557,16 @@ mod tests {
         };
 
         create(&server, &table, 4).await;
+        create(&server, &table, 4).await;
+        let mismatch = server
+            .create_index(Request::new(pb::CreateIndexRequest {
+                table: Some(table.clone()),
+                dimension: 3,
+                ..Default::default()
+            }))
+            .await
+            .unwrap_err();
+        assert_eq!(mismatch.code(), tonic::Code::AlreadyExists);
         put(
             &server,
             &table,

seaweed-vector/crates/server/src/service.rs

@@ -95,6 +95,44 @@ fn schema_for(
     ]))
 }
 
+fn matches_index_schema(
+    dataset: &Dataset,
+    dimension: usize,
+    metric: pb::DistanceMetric,
+    non_filterable: &[String],
+) -> bool {
+    let Ok(layout) = VectorTable::resolve(dataset, Some(VECTOR_COLUMN), Some(KEY_COLUMN)) else {
+        return false;
+    };
+    let schema = Schema::from(dataset.schema());
+    let Ok(vector) = schema.field_with_name(VECTOR_COLUMN) else {
+        return false;
+    };
+    let Ok(metadata) = schema.field_with_name(METADATA_COLUMN) else {
+        return false;
+    };
+    let stored_keys = metadata
+        .metadata()
+        .get("non_filterable_metadata_keys")
+        .map(|keys| serde_json::from_str::<Vec<String>>(keys))
+        .transpose();
+    let Ok(stored_keys) = stored_keys else {
+        return false;
+    };
+    let mut stored_keys = stored_keys.unwrap_or_default();
+    let mut requested_keys = non_filterable.to_vec();
+    stored_keys.sort();
+    requested_keys.sort();
+    layout.dimension == dimension
+        && vector.metadata().get("distance_metric").map(String::as_str)
+            == Some(metric.as_str_name())
+        && stored_keys == requested_keys
+        && metadata
+            .metadata()
+            .get("ARROW:extension:name")
+            .is_some_and(|name| name == "lance.json")
+}
+
 fn batch_of(vectors: &[pb::Vector], dimension: usize, schema: Arc<Schema>) -> Result<RecordBatch> {
     let mut seen = std::collections::HashSet::new();
     for vector in vectors {
@@ -536,8 +574,28 @@ impl VectorService for VectorServer {
                 ..Default::default()
             }),
         )
-        .await
-        .map_err(internal)?;
+        .await;
+        let dataset = match dataset {
+            Ok(dataset) => dataset,
+            Err(error) => match self.open(&table).await {
+                Ok(existing)
+                    if matches_index_schema(
+                        &existing,
+                        dimension,
+                        metric,
+                        &req.non_filterable_metadata_keys,
+                    ) =>
+                {
+                    existing
+                }
+                Ok(_) => {
+                    return Err(Status::already_exists(
+                        "index exists with a different schema",
+                    ));
+                }
+                Err(_) => return Err(internal(error)),
+            },
+        };
         Ok(Response::new(pb::CreateIndexResponse {
             version: dataset.version().version,
         }))

seaweed-vector/crates/server/tests/cache_metrics.rs

@@ -0,0 +1,111 @@
+use std::sync::Arc;
+
+use bytes::Bytes;
+use lance::io::WrappingObjectStore;
+use object_store::{ObjectStoreExt, memory::InMemory, path::Path};
+use weed_vector_server::{metrics::Metrics, read_cache::ReadCache};
+
+#[tokio::test]
+async fn immutable_reads_report_hits_misses_bytes_and_weight() {
+    let metrics = Metrics::new().unwrap();
+    let cache = ReadCache::new(1024).with_metrics(metrics.clone());
+    let store = cache.wrap("memory", Arc::new(InMemory::new()));
+    let path = Path::from("data/example.lance");
+    store
+        .put(&path, Bytes::from_static(b"abc").into())
+        .await
+        .unwrap();
+    for _ in 0..2 {
+        assert_eq!(
+            store.get(&path).await.unwrap().bytes().await.unwrap(),
+            "abc"
+        );
+    }
+    assert_eq!(metrics.cache_requests.with_label_values(&["miss"]).get(), 1);
+    assert_eq!(metrics.cache_requests.with_label_values(&["hit"]).get(), 1);
+    assert_eq!(metrics.storage_bytes.get(), 3);
+    assert_eq!(metrics.storage_read_seconds.get_sample_count(), 1);
+    let (entries, weight) = cache.size();
+    assert_eq!(entries, 1);
+    assert!(weight >= 3);
+    let metrics_endpoint = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let endpoint = format!("http://{}/metrics", metrics_endpoint.local_addr().unwrap());
+    let task = tokio::spawn(weed_vector_server::metrics::serve_listener(
+        metrics,
+        metrics_endpoint,
+        Some(cache),
+    ));
+    let body = reqwest::get(endpoint).await.unwrap().text().await.unwrap();
+    assert!(body.contains("SeaweedFS_vector_cache_entries 1"));
+    assert!(body.contains("SeaweedFS_vector_cache_bytes "));
+    task.abort();
+}
+
+#[tokio::test]
+async fn cancelled_requests_are_counted_once() {
+    let metrics = Metrics::new().unwrap();
+    let guard = metrics.start("QueryVectors");
+    let task = tokio::spawn(async move {
+        let _guard = guard;
+        std::future::pending::<()>().await;
+    });
+    task.abort();
+    let _ = task.await;
+    assert_eq!(
+        metrics.in_flight.with_label_values(&["QueryVectors"]).get(),
+        0
+    );
+    assert_eq!(
+        metrics
+            .requests
+            .with_label_values(&["QueryVectors", "Cancelled"])
+            .get(),
+        1
+    );
+    assert_eq!(
+        metrics
+            .request_seconds
+            .with_label_values(&["QueryVectors"])
+            .get_sample_count(),
+        1
+    );
+    {
+        let mut guard = metrics.start("QueryVectors");
+        guard.complete(None);
+    }
+    assert_eq!(
+        metrics
+            .requests
+            .with_label_values(&["QueryVectors", "OK"])
+            .get(),
+        1
+    );
+    assert_eq!(
+        metrics
+            .requests
+            .with_label_values(&["QueryVectors", "Cancelled"])
+            .get(),
+        1
+    );
+}
+
+#[tokio::test]
+async fn large_immutable_reads_report_bytes_without_caching() {
+    let metrics = Metrics::new().unwrap();
+    let cache = ReadCache::new(16 * 1024 * 1024).with_metrics(metrics.clone());
+    let store = cache.wrap("memory", Arc::new(InMemory::new()));
+    let path = Path::from("data/large.lance");
+    let size = 5 * 1024 * 1024;
+    store
+        .put(&path, Bytes::from(vec![0; size]).into())
+        .await
+        .unwrap();
+    let result = store.get(&path).await.unwrap();
+    assert_eq!(metrics.storage_bytes.get(), 0);
+    assert_eq!(metrics.storage_read_seconds.get_sample_count(), 0);
+    assert_eq!(result.bytes().await.unwrap().len(), size);
+    assert_eq!(metrics.storage_bytes.get(), size as u64);
+    assert_eq!(metrics.cache_requests.with_label_values(&["miss"]).get(), 1);
+    assert_eq!(metrics.storage_read_seconds.get_sample_count(), 1);
+    assert_eq!(cache.size(), (0, 0));
+}

test/s3vectors/conftest.py

@@ -123,7 +123,7 @@ def write_tls(directory):
             "-out", "outsider.pem", "-days", "2", "-extfile", "ext.cnf")
     cert, key, ca = (f'{key} = {json.dumps(str(directory / name), ensure_ascii=False)}\n' for key, name in (("cert", "cert.pem"), ("key", "key.pem"), ("ca", "ca.pem")))
     pem = cert + key + ca
-    grpc = "".join(f"[grpc.{component}]\n{cert}{key}\n" for component in ("master", "volume", "filer", "s3"))
+    grpc = "".join(f"[grpc.{component}]\n{cert}{key}\n" for component in ("master", "volume", "filer", "s3", "admin", "worker"))
     # Clients (the gateway, the engine) accept only servers named localhost;
     # the engine serves only clients named localhost.
     grpc += f"[grpc.client]\n{cert}{key}allowed_server_commonNames = \"localhost\"\n"
@@ -248,12 +248,12 @@ def run_cluster(param, tmp_path_factory, identities):
     with contextlib.ExitStack() as stack:
         with contextlib.ExitStack() as reservations:
             ports = []
-            for _ in range(14):
+            for _ in range(15):
                 sock = reservations.enter_context(socket.socket())
                 sock.bind(("127.0.0.1", 0))
                 ports.append(sock.getsockname()[1])
         (master, master_grpc, vol, vol_grpc, filer, filer_grpc, s3, s3_grpc, lance, vector, worker1, worker2,
-         admin, admin_grpc) = ports
+         admin, admin_grpc, vector_metrics) = ports
         master_addr = f"127.0.0.1:{master}.{master_grpc}"
         filer_addr = f"127.0.0.1:{filer}.{filer_grpc}"
 
@@ -304,10 +304,11 @@ def run_cluster(param, tmp_path_factory, identities):
         # VECTOR_ENCRYPT_VOLUMES=1 turns on the filer's volume encryption, which
         # the gateway, the engine and the worker all follow: every chunk they
         # store is encrypted, and direct reads must undo it.
+        filer_group = os.environ.get("VECTOR_FILER_GROUP", "")
         encrypt = ["-encryptVolumeData"] if os.environ.get("VECTOR_ENCRYPT_VOLUMES") else []
         start("filer", [weed, "filer", f"-master={master_addr}", "-ip=127.0.0.1",
                         "-ip.bind=127.0.0.1", f"-port={filer}", f"-port.grpc={filer_grpc}",
-                        f"-defaultStoreDir={directory / 'filer'}", *encrypt])
+                        f"-defaultStoreDir={directory / 'filer'}", f"-filerGroup={filer_group}", *encrypt])
         wait_for(lambda: listening(filer_grpc), processes, "filer not ready")
         endpoint = f"http://127.0.0.1:{s3}"
         # weed s3 turns AWS_ACCESS_KEY_ID into an admin identity.
@@ -325,8 +326,12 @@ def run_cluster(param, tmp_path_factory, identities):
         # The engine keeps tables on the filers and volume servers itself; it
         # has no S3 credentials to be given, and encrypts its chunks for the
         # volume servers as the gateway does.
-        start("vector", [engine, "--masters", master_addr, "--listen", f"127.0.0.1:{vector}"])
+        start("vector", [engine, "--masters", master_addr, "--listen", f"127.0.0.1:{vector}", "--metrics-port", str(vector_metrics), "--filer-group", filer_group])
         wait_for(lambda: listening(vector), processes, "vector engine not ready")
+        def vector_ready():
+            with urllib.request.urlopen(f"http://127.0.0.1:{vector_metrics}/ready", timeout=1) as response:
+                return response.status == 200
+        wait_for(vector_ready, processes, "vector engine did not register with master")
         if os.environ.get("VECTOR_TLS"):
             wait_for(lambda: listening(s3_grpc), processes, "s3 gRPC not ready")
             for name, port in (("master", master_grpc), ("volume", vol_grpc), ("filer", filer_grpc),
@@ -335,7 +340,7 @@ def run_cluster(param, tmp_path_factory, identities):
             probe_grpc_tls("engine", vector, directory, outsider=True)
 
         wait_for(
-            lambda: f"Cluster: vector node 127.0.0.1:{vector} added to group ''"
+            lambda: f"Cluster: vector node 127.0.0.1:{vector} added to group '{filer_group}'"
             in (directory / "master.log").read_text(),
             processes,
             "vector engine never registered with the master",
@@ -358,18 +363,22 @@ def run_cluster(param, tmp_path_factory, identities):
         if lance_worker:
             (directory / "admin").mkdir()
             admin_url = f"http://127.0.0.1:{admin}"
-            start("admin", [weed, "admin", "-ip=127.0.0.1", f"-port={admin}", f"-port.grpc={admin_grpc}",
+            start("admin", [weed, "admin", f"-filerGroup={filer_group}", "-ip=127.0.0.1", f"-port={admin}", f"-port.grpc={admin_grpc}",
                             "-port.legacy=0", f"-master={master_addr}", f"-dataDir={directory / 'admin'}"])
             wait_for(lambda: http_ready(f"{admin_url}/api/plugin/job-types"), processes, "admin not ready")
+            tls_args = []
+            if os.environ.get("VECTOR_TLS"):
+                tls_args = ["--tls-ca", str(directory / "ca.pem"), "--tls-cert", str(directory / "cert.pem"), "--tls-key", str(directory / "key.pem"), "--tls-server-name", "localhost"]
             start("lance-worker", [lance_worker, "--admin", f"127.0.0.1:{admin}.{admin_grpc}",
                                    "--namespace", f"http://127.0.0.1:{lance}", "--id", "vectors-test-worker",
-                                   "--masters", master_addr])
+                                   "--masters", master_addr, "--filer-group", filer_group, *tls_args])
             wait_for(lambda: "lance_vector_index" in
                      [jt["job_type"] for jt in json.load(urllib.request.urlopen(f"{admin_url}/api/plugin/job-types", timeout=2))],
                      processes, "the Lance worker never registered with admin")
 
         yield {"endpoint": endpoint, "mode": mode, "processes": processes, "directory": directory, "filer": filer,
                "masters": master_addr, "filer_signing_key": filer_signing_key,
+               "vector_metrics": f"http://127.0.0.1:{vector_metrics}",
                "catalog": f"http://127.0.0.1:{lance}", "admin": admin_url, "env": process_env()}
 
 

test/s3vectors/s3.json

@@ -39,6 +39,24 @@
       "actions": [
         "Write"
       ]
+    },
+    {
+      "name": "policy-reader",
+      "credentials": [
+        {
+          "accessKey": "policy-reader",
+          "secretKey": "policy-reader-secret"
+        }
+      ],
+      "policyNames": [
+        "vector-policy"
+      ]
+    }
+  ],
+  "policies": [
+    {
+      "name": "vector-policy",
+      "content": "{\"Version\": \"2012-10-17\", \"Statement\": [{\"Effect\": \"Allow\", \"Action\": [\"s3vectors:GetVectors\", \"s3vectors:QueryVectors\"], \"Resource\": \"arn:aws:s3vectors:*:*:bucket/vector-iam/index/docs\"}, {\"Effect\": \"Allow\", \"Action\": \"s3vectors:CreateVectorBucket\", \"Resource\": \"*\"}, {\"Effect\": \"Allow\", \"Action\": \"s3vectors:TagResource\", \"Resource\": \"arn:aws:s3vectors:*:*:bucket/vector-iam-*\"}, {\"Effect\": \"Deny\", \"Action\": \"s3vectors:QueryVectors\", \"Resource\": \"*\", \"Condition\": {\"StringEquals\": {\"aws:ResourceTag/classification\": \"private\"}}}, {\"Effect\": \"Deny\", \"Action\": \"s3vectors:QueryVectors\", \"Resource\": \"*\", \"Condition\": {\"StringEquals\": {\"s3vectors:VectorBucketTag/environment\": \"production\"}}}, {\"Effect\": \"Deny\", \"Action\": \"s3vectors:CreateVectorBucket\", \"Resource\": \"*\", \"Condition\": {\"StringEquals\": {\"aws:RequestTag/classification\": \"forbidden\"}}}, {\"Effect\": \"Deny\", \"Action\": \"s3vectors:CreateVectorBucket\", \"Resource\": \"*\", \"Condition\": {\"ForAnyValue:StringEquals\": {\"aws:TagKeys\": \"forbidden-key\"}}}]}"
     }
   ]
 }

test/s3vectors/s3vectors_test.py

@@ -1,3 +1,4 @@
+import urllib.request
 """Run the S3 Vectors API against Go/Rust volumes and single/distributed queries."""
 
 import contextlib
@@ -150,7 +151,7 @@ def run_cli(cluster, *args):
     from pathlib import Path
     cli = Path(os.environ.get("VECTOR_CLI_BINARY", Path(__file__).resolve().parent / "weed-vector-cli")).resolve()
     command = [str(cli), "--namespace", cluster["catalog"], "--masters", cluster["masters"],
-               "--filer-signing-key", cluster["filer_signing_key"], *args]
+               "--filer-signing-key", cluster["filer_signing_key"], "--filer-group", os.environ.get("VECTOR_FILER_GROUP", ""), *args]
     return json.loads(subprocess.run(command, check=True, capture_output=True, text=True,
                                      cwd=cluster["directory"], env=cluster["env"]).stdout)
 
@@ -711,6 +712,43 @@ def test_vector_index_job(vectors, cluster):
     assert found >= 180, f"recall after retrain {found}/200"
 
 
+def test_attached_iam_vector_policies(vectors, cluster):
+    client = boto3.client("s3vectors", endpoint_url=cluster["endpoint"], region_name="us-east-1",
+                          aws_access_key_id="policy-reader", aws_secret_access_key="policy-reader-secret",
+                          config=vectors.meta.config)
+    bucket = "vector-iam"
+    vectors.create_vector_bucket(vectorBucketName=bucket)
+    target = {"vectorBucketName": bucket, "indexName": "docs"}
+    vectors.create_index(**target, dataType="float32", dimension=3, distanceMetric="cosine")
+    index_arn = vectors.get_index(**target)["index"]["indexArn"]
+    bucket_arn = vectors.get_vector_bucket(vectorBucketName=bucket)["vectorBucket"]["vectorBucketArn"]
+    query = {**target, "queryVector": {"float32": [1, 0, 0]}, "topK": 1}
+    try:
+        assert client.query_vectors(**query)["vectors"] == []
+        assert client.get_vectors(**target, keys=["missing"])["vectors"] == []
+        expect_error("AccessDeniedException", client.put_vectors, **target,
+                     vectors=[{"key": "blocked", "data": {"float32": [1, 0, 0]}}])
+        vectors.tag_resource(resourceArn=index_arn, tags={"classification": "private"})
+        expect_error("AccessDeniedException", client.query_vectors, **query)
+        vectors.untag_resource(resourceArn=index_arn, tagKeys=["classification"])
+        assert client.query_vectors(**query)["vectors"] == []
+        vectors.tag_resource(resourceArn=bucket_arn, tags={"environment": "production"})
+        expect_error("AccessDeniedException", client.query_vectors, **query)
+        vectors.untag_resource(resourceArn=bucket_arn, tagKeys=["environment"])
+        assert client.query_vectors(**query)["vectors"] == []
+        vectors.put_vector_bucket_policy(vectorBucketName=bucket, policy=json.dumps({"Version": "2012-10-17", "Statement": [
+            {"Effect": "Deny", "Principal": "*", "Action": "s3vectors:QueryVectors", "Resource": index_arn}]}))
+        expect_error("AccessDeniedException", client.query_vectors, **query)
+        expect_error("AccessDeniedException", client.create_vector_bucket,
+                     vectorBucketName="vector-iam-rejected", tags={"classification": "forbidden"})
+        expect_error("AccessDeniedException", client.create_vector_bucket,
+                     vectorBucketName="vector-iam-key-denied", tags={"forbidden-key": "value"})
+    finally:
+        vectors.delete_vector_bucket_policy(vectorBucketName=bucket)
+        vectors.delete_index(**target)
+        vectors.delete_vector_bucket(vectorBucketName=bucket)
+
+
 def test_invalid_credentials(cluster):
     client = boto3.client("s3vectors", endpoint_url=cluster["endpoint"], aws_access_key_id="unknown",
                           aws_secret_access_key="unknown", region_name="us-east-1")
@@ -719,6 +757,18 @@ def test_invalid_credentials(cluster):
     assert exc.value.response["ResponseMetadata"]["HTTPStatusCode"] == 403
 
 
+def test_vector_engine_observability(vectors, index, cluster):
+    vectors.query_vectors(**index, queryVector={"float32": [1, 0, 0, 0]}, topK=1)
+    with urllib.request.urlopen(cluster["vector_metrics"] + "/ready", timeout=5) as response:
+        assert response.status == 200
+    with urllib.request.urlopen(cluster["vector_metrics"] + "/metrics", timeout=5) as response:
+        metrics = response.read().decode()
+    assert "SeaweedFS_vector_connected 1" in metrics
+    assert any(line.startswith("SeaweedFS_vector_request_total{") and 'type="QueryVectors"' in line
+               and 'code="OK"' in line for line in metrics.splitlines())
+    assert "SeaweedFS_vector_request_seconds_bucket" in metrics
+
+
 def test_query_without_writer(vectors, index, cluster):
     from conftest import stop
 
@@ -753,3 +803,5 @@ def test_engine_without_credentials(open_cluster):
     hits = client.query_vectors(**index, queryVector={"float32": [1, 0]}, topK=1)["vectors"]
     assert [h["key"] for h in hits] == ["a"]
 
+
+

weed/admin/dash/admin_data.go

@@ -32,6 +32,7 @@ type AdminData struct {
 	VolumeServers     []VolumeServer      `json:"volume_servers"`
 	FilerNodes        []FilerNode         `json:"filer_nodes"`
 	MessageBrokers    []MessageBrokerNode `json:"message_brokers"`
+	VectorNodes       []VectorNode        `json:"vector_nodes"`
 	S3Nodes           []S3Node            `json:"s3_nodes"`
 	DataCenters       []DataCenter        `json:"datacenters"`
 	LastUpdated       time.Time           `json:"last_updated"`
@@ -250,6 +251,7 @@ func (s *AdminServer) GetAdminData(username string) (AdminData, error) {
 		FilerNodes:        filerNodes,
 		MessageBrokers:    messageBrokers,
 		S3Nodes:           s3Nodes,
+		VectorNodes:       s.getVectorNodesStatus(context.Background()).Engines,
 		DataCenters:       topology.DataCenters,
 		LastUpdated:       topology.UpdatedAt,
 		TotalEcVolumes:    totalEcVolumes,

weed/admin/dash/admin_server_filergroup_test.go

@@ -28,3 +28,11 @@ func TestListClusterNodesRequestKeepsDefaultFilerGroupEmpty(t *testing.T) {
 		t.Fatalf("FilerGroup = %q, want empty", req.FilerGroup)
 	}
 }
+
+func TestVectorDiscoveryUsesRegistrationGroup(t *testing.T) {
+	s := &AdminServer{filerGroup: "tenant-a"}
+	request := s.listClusterNodesRequest(cluster.VectorType)
+	if request.FilerGroup != s.filerGroup {
+		t.Fatalf("vector registration group must match the filer: %+v", request)
+	}
+}

weed/admin/dash/s3vectors_management.go

@@ -2,11 +2,13 @@ package dash
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"net/http"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/lance"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
@@ -15,6 +17,7 @@ import (
 type S3VectorsBucketsData struct {
 	Username     string                   `json:"username"`
 	Buckets      []S3VectorsBucketSummary `json:"buckets"`
+	Operations   VectorOperationsData     `json:"operations"`
 	TotalBuckets int                      `json:"total_buckets"`
 	Truncated    bool                     `json:"truncated,omitempty"`
 	LastUpdated  time.Time                `json:"last_updated"`
@@ -33,6 +36,7 @@ type S3VectorsBucketData struct {
 	Bucket       S3VectorsBucketSummary `json:"bucket"`
 	Policy       string                 `json:"policy,omitempty"`
 	Indexes      []*s3vectors.Index     `json:"indexes"`
+	Operations   VectorOperationsData   `json:"operations"`
 	TotalIndexes int                    `json:"total_indexes"`
 	Truncated    bool                   `json:"truncated,omitempty"`
 	LastUpdated  time.Time              `json:"last_updated"`
@@ -46,11 +50,15 @@ func (c s3VectorsFilerClient) WithFilerClient(_ bool, fn func(client filer_pb.Se
 	return c.s.WithFilerClient(fn)
 }
 
-// newS3VectorsServer serves the catalog side of S3 Vectors for the admin
-// console. It has no vector engine, so it never runs an operation that needs
-// one, such as CreateIndex.
 func newS3VectorsServer(s *AdminServer) *s3vectors.Server {
-	server := s3vectors.NewServer(s3VectorsFilerClient{s}, nil, nil)
+	client := s3VectorsFilerClient{s}
+	backend := s3vectors.NewVectorDiscovery(s.masterClient.ListCachedMasters, s.grpcDialOption, s.filerGroup)
+	server := s3vectors.NewServer(client, nil, backend)
+	catalog := lance.NewServer(client, nil)
+	server.SetDescribe(func(ctx context.Context, id []string) (string, map[string]string, error) {
+		request := (&http.Request{Method: http.MethodPost, Header: make(http.Header)}).WithContext(ctx)
+		return catalog.DescribeTableInternal(request, id)
+	})
 	server.SetRegion(s3tables.DefaultRegion)
 	server.SetAccountID(s3tables.DefaultAccountID)
 	return server
@@ -103,6 +111,7 @@ func (s *AdminServer) GetS3VectorsBucketsData(ctx context.Context) (S3VectorsBuc
 		}
 		req.NextToken = resp.NextToken
 	}
+	data.Operations = s.GetVectorOperationsData(ctx)
 	data.TotalBuckets = len(data.Buckets)
 	data.LastUpdated = time.Now()
 	return data, nil
@@ -148,6 +157,7 @@ func (s *AdminServer) GetS3VectorsBucketData(ctx context.Context, bucket string)
 		}
 		req.NextToken = resp.NextToken
 	}
+	data.Operations = s.GetVectorOperationsData(ctx)
 	data.TotalIndexes = len(data.Indexes)
 	return data, nil
 }
@@ -166,14 +176,20 @@ func (s *AdminServer) CreateS3VectorsBucket(w http.ResponseWriter, r *http.Reque
 		return
 	}
 	var req struct {
-		Name string `json:"name"`
+		Name                    string                 `json:"name"`
+		EncryptionConfiguration map[string]interface{} `json:"encryptionConfiguration,omitempty"`
+		Tags                    map[string]string      `json:"tags,omitempty"`
 	}
 	if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
 		writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
 		return
 	}
+	if req.EncryptionConfiguration["sseType"] == "AES256" && !s.vectorEncryptionConfigured(r.Context()) {
+		writeJSONError(w, http.StatusBadRequest, "Cannot verify SSE-S3 support on the S3 gateway; configure s3.public_endpoint and an SSE-S3 key on the gateway and vector engines")
+		return
+	}
 	var resp s3vectors.CreateVectorBucketResponse
-	if err := s.executeS3VectorsOperation(r.Context(), "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: req.Name}, &resp); err != nil {
+	if err := s.executeS3VectorsOperation(r.Context(), "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: req.Name, EncryptionConfiguration: req.EncryptionConfiguration, Tags: req.Tags}, &resp); err != nil {
 		writeS3VectorsError(w, err)
 		return
 	}
@@ -252,3 +268,40 @@ func writeS3VectorsError(w http.ResponseWriter, err error) {
 	}
 	writeJSONError(w, status, e.Message)
 }
+
+func (s *AdminServer) ExecuteS3VectorsOperationAPI(w http.ResponseWriter, r *http.Request) {
+	if !requireSessionCSRFToken(w, r) {
+		return
+	}
+	var req struct {
+		Operation string          `json:"operation"`
+		Request   json.RawMessage `json:"request"`
+	}
+	if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
+		writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
+		return
+	}
+	switch req.Operation {
+	case "CreateIndex", "PutVectorBucketPolicy", "DeleteVectorBucketPolicy", "TagResource", "UntagResource", "ListTagsForResource", "PutVectors", "GetVectors", "ListVectors", "DeleteVectors", "QueryVectors":
+	default:
+		writeJSONError(w, http.StatusBadRequest, "Unsupported vector operation")
+		return
+	}
+	switch req.Operation {
+	case "QueryVectors", "GetVectors", "ListVectors", "ListTagsForResource":
+	default:
+		if role := RoleFromContext(r.Context()); role != "admin" && (role != "" || UsernameFromContext(r.Context()) != "") {
+			writeJSONError(w, http.StatusForbidden, "This operation requires admin access")
+			return
+		}
+	}
+	var resp json.RawMessage
+	if err := s.executeS3VectorsOperation(r.Context(), req.Operation, req.Request, &resp); err != nil {
+		writeS3VectorsError(w, err)
+		return
+	}
+	if len(resp) == 0 {
+		resp = json.RawMessage(`{}`)
+	}
+	writeJSON(w, http.StatusOK, resp)
+}

weed/admin/dash/s3vectors_management_test.go

@@ -22,7 +22,9 @@ func newS3VectorsTestServer(t *testing.T) (*AdminServer, *s3tables.ManagerClient
 	server := s3vectors.NewServer(client, nil, nil)
 	server.SetRegion(s3tables.DefaultRegion)
 	server.SetAccountID(s3tables.DefaultAccountID)
-	return &AdminServer{s3VectorsServer: server}, client
+	gateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(`{"sseS3":true}`)) }))
+	t.Cleanup(gateway.Close)
+	return &AdminServer{s3VectorsServer: server, s3PublicEndpoint: gateway.URL}, client
 }
 
 // createS3VectorsIndexRecord writes the catalog record CreateIndex leaves
@@ -172,3 +174,79 @@ func TestS3VectorsBucketsStopAtTheListLimit(t *testing.T) {
 		t.Fatalf("listed %d buckets (truncated %v), want the limit %d and truncated", data.TotalBuckets, data.Truncated, s3TablesAdminListLimit)
 	}
 }
+
+func TestS3VectorsManagementOperations(t *testing.T) {
+	s, _ := newS3VectorsTestServer(t)
+	w := s3VectorsRequest(t, s.CreateS3VectorsBucket, http.MethodPost, "/api/s3vectors/buckets", `{"name":"managed","encryptionConfiguration":{"sseType":"AES256"},"tags":{"env":"test"}}`)
+	if w.Code != http.StatusCreated {
+		t.Fatal(w.Code, w.Body)
+	}
+	summary, err := s.s3VectorsBucketSummary(t.Context(), "managed")
+	if err != nil || summary.Encryption != "AES256" {
+		t.Fatal(summary, err)
+	}
+	arn := summary.ARN
+	for _, body := range []string{
+		`{"operation":"TagResource","request":{"resourceArn":"` + arn + `","tags":{"team":"search"}}}`,
+		`{"operation":"ListTagsForResource","request":{"resourceArn":"` + arn + `"}}`,
+		`{"operation":"UntagResource","request":{"resourceArn":"` + arn + `","tagKeys":["team"]}}`,
+		`{"operation":"PutVectorBucketPolicy","request":{"vectorBucketName":"managed","policy":"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":\"*\",\"Action\":\"s3vectors:QueryVectors\",\"Resource\":\"*\"}]}"}}`,
+		`{"operation":"DeleteVectorBucketPolicy","request":{"vectorBucketName":"managed"}}`,
+	} {
+		w := s3VectorsRequest(t, s.ExecuteS3VectorsOperationAPI, http.MethodPost, "/api/s3vectors/operations", body)
+		if w.Code != http.StatusOK {
+			t.Fatalf("%s: %d %s", body, w.Code, w.Body)
+		}
+	}
+	for _, body := range []string{`{"operation":"Unknown","request":{}}`, `{bad json`} {
+		if w := s3VectorsRequest(t, s.ExecuteS3VectorsOperationAPI, http.MethodPost, "/api/s3vectors/operations", body); w.Code != http.StatusBadRequest {
+			t.Fatal(w.Code, w.Body)
+		}
+	}
+	request := httptest.NewRequest(http.MethodPost, "/api/s3vectors/operations", strings.NewReader(`{}`))
+	request = request.WithContext(WithAuthContext(request.Context(), "admin", "admin", "required-token"))
+	recorder := httptest.NewRecorder()
+	s.ExecuteS3VectorsOperationAPI(recorder, request)
+	if recorder.Code != http.StatusForbidden {
+		t.Fatal("missing CSRF token accepted", recorder.Code)
+	}
+}
+
+func TestS3VectorsEncryptedCreationRequiresKey(t *testing.T) {
+	s, _ := newS3VectorsTestServer(t)
+	s.s3VectorsServer.SetEncryptionChecker(func() bool { return false })
+	w := s3VectorsRequest(t, s.CreateS3VectorsBucket, http.MethodPost, "/api/s3vectors/buckets", `{"name":"sealed","encryptionConfiguration":{"sseType":"AES256"}}`)
+	if w.Code != http.StatusBadRequest {
+		t.Fatalf("encrypted bucket without key: %d %s", w.Code, w.Body)
+	}
+	data, err := s.GetS3VectorsBucketsData(t.Context())
+	if err != nil || len(data.Buckets) != 0 {
+		t.Fatalf("unusable bucket persisted: %+v %v", data, err)
+	}
+}
+
+func TestS3VectorsReadOnlyOperations(t *testing.T) {
+	s, _ := newS3VectorsTestServer(t)
+	s3VectorsRequest(t, s.CreateS3VectorsBucket, http.MethodPost, "/api/s3vectors/buckets", `{"name":"readable"}`)
+	summary, err := s.s3VectorsBucketSummary(t.Context(), "readable")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, role := range []string{"readonly", ""} {
+		for _, operation := range []string{"ListTagsForResource", "TagResource", "CreateIndex", "DeleteVectors"} {
+			body := fmt.Sprintf(`{"operation":%q,"request":{"resourceArn":%q}}`, operation, summary.ARN)
+			request := httptest.NewRequest(http.MethodPost, "/api/s3vectors/operations", strings.NewReader(body))
+			request = request.WithContext(WithAuthContext(request.Context(), "viewer", role, "token"))
+			request.Header.Set("X-CSRF-Token", "token")
+			w := httptest.NewRecorder()
+			s.ExecuteS3VectorsOperationAPI(w, request)
+			want := http.StatusForbidden
+			if operation == "ListTagsForResource" {
+				want = http.StatusOK
+			}
+			if w.Code != want {
+				t.Fatalf("%s: %d %s", operation, w.Code, w.Body)
+			}
+		}
+	}
+}

weed/admin/dash/vector_encryption.go

@@ -0,0 +1,60 @@
+package dash
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+func (s *AdminServer) vectorEncryptionConfigured(ctx context.Context) bool {
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	endpoints := []string{}
+	if s.s3PublicEndpoint != "" {
+		endpoints = append(endpoints, s.s3PublicEndpoint)
+	} else if s.masterClient != nil {
+		err := s.masterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {
+			response, err := client.ListClusterNodes(ctx, s.listClusterNodesRequest(cluster.S3Type))
+			if err != nil {
+				return err
+			}
+			for _, node := range response.ClusterNodes {
+				endpoints = append(endpoints, "http://"+pb.ServerAddress(node.Address).ToHttpAddress())
+			}
+			return nil
+		})
+		if err != nil {
+			return false
+		}
+	}
+	if len(endpoints) == 0 {
+		return false
+	}
+	for _, endpoint := range endpoints {
+		request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(endpoint, "/")+"/status?features", nil)
+		if err != nil {
+			return false
+		}
+		response, err := util_http.GetGlobalHttpClient().Do(request)
+		if err != nil {
+			return false
+		}
+		var features struct {
+			SSES3 bool `json:"sseS3"`
+		}
+		err = json.NewDecoder(io.LimitReader(response.Body, 4096)).Decode(&features)
+		_ = response.Body.Close()
+		if response.StatusCode != http.StatusOK || err != nil || !features.SSES3 {
+			return false
+		}
+	}
+	return true
+}

weed/admin/dash/vector_encryption_test.go

@@ -0,0 +1,26 @@
+package dash
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestVectorEncryptionChecksRemoteGateway(t *testing.T) {
+	for _, test := range []struct {
+		body string
+		want bool
+	}{{`{"sseS3":true}`, true}, {`{"sseS3":false}`, false}, {``, false}} {
+		gateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.URL.Path != "/status" || !r.URL.Query().Has("features") {
+				t.Errorf("wrong probe: %s", r.URL)
+			}
+			_, _ = w.Write([]byte(test.body))
+		}))
+		s := &AdminServer{s3PublicEndpoint: gateway.URL}
+		if got := s.vectorEncryptionConfigured(t.Context()); got != test.want {
+			t.Errorf("body %q: %v", test.body, got)
+		}
+		gateway.Close()
+	}
+}

weed/admin/dash/vector_status.go

@@ -0,0 +1,107 @@
+package dash
+
+import (
+	"context"
+	"sort"
+	"time"
+
+	adminplugin "github.com/seaweedfs/seaweedfs/weed/admin/plugin"
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/health/grpc_health_v1"
+	"google.golang.org/grpc/status"
+)
+
+type VectorNode struct {
+	Address string `json:"address"`
+	Version string `json:"version"`
+	Status  string `json:"status"`
+}
+
+type VectorOperationsData struct {
+	Engines            []VectorNode             `json:"engines"`
+	DiscoveryError     string                   `json:"discovery_error,omitempty"`
+	MaintenanceWorkers int                      `json:"maintenance_workers"`
+	MaintenanceJobs    []adminplugin.TrackedJob `json:"maintenance_jobs"`
+}
+
+func (s *AdminServer) getVectorNodesStatus(ctx context.Context) VectorOperationsData {
+	data := VectorOperationsData{Engines: []VectorNode{}}
+	if s.masterClient == nil {
+		data.DiscoveryError = "Master discovery unavailable"
+		return data
+	}
+	var discoveryErr error
+	for _, master := range s.masterClient.ListCachedMasters() {
+		discoveryCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		conn, err := pb.GrpcDial(discoveryCtx, master.ToGrpcAddress(), false, s.grpcDialOption)
+		if err == nil {
+			response, callErr := master_pb.NewSeaweedClient(conn).ListClusterNodes(discoveryCtx, s.listClusterNodesRequest(cluster.VectorType))
+			_ = conn.Close()
+			err = callErr
+			if err == nil {
+				for _, node := range response.ClusterNodes {
+					data.Engines = append(data.Engines, VectorNode{Address: node.Address, Version: node.Version, Status: "Registered"})
+				}
+			}
+		}
+		cancel()
+		if err != nil {
+			discoveryErr = err
+		}
+		if len(data.Engines) > 0 {
+			break
+		}
+	}
+	if len(data.Engines) == 0 && discoveryErr != nil {
+		data.DiscoveryError = discoveryErr.Error()
+		return data
+	}
+	var checks errgroup.Group
+	checks.SetLimit(8)
+	for i := range data.Engines {
+		checks.Go(func() error {
+			data.Engines[i].Status = vectorNodeStatus(ctx, data.Engines[i].Address, s.grpcDialOption)
+			return nil
+		})
+	}
+	_ = checks.Wait()
+	sort.Slice(data.Engines, func(i, j int) bool { return data.Engines[i].Address < data.Engines[j].Address })
+	return data
+}
+
+func vectorNodeStatus(ctx context.Context, address string, dialOption grpc.DialOption) string {
+	ctx, cancel := context.WithTimeout(ctx, time.Second)
+	defer cancel()
+	conn, err := pb.GrpcDial(ctx, address, false, dialOption)
+	if err != nil {
+		return "Unreachable"
+	}
+	defer conn.Close()
+	response, err := grpc_health_v1.NewHealthClient(conn).Check(ctx, &grpc_health_v1.HealthCheckRequest{Service: "vector_pb.VectorService"})
+	if status.Code(err) == codes.Unimplemented || status.Code(err) == codes.NotFound {
+		return "Registered (health unavailable)"
+	}
+	if err != nil {
+		return "Unreachable"
+	}
+	if response.Status == grpc_health_v1.HealthCheckResponse_SERVING {
+		return "Ready"
+	}
+	return "Not ready"
+}
+
+func (s *AdminServer) GetVectorOperationsData(ctx context.Context) VectorOperationsData {
+	data := s.getVectorNodesStatus(ctx)
+	for _, worker := range s.GetPluginWorkers() {
+		if capability := worker.Capabilities["lance_vector_index"]; capability != nil && capability.CanExecute {
+			data.MaintenanceWorkers++
+		}
+	}
+	data.MaintenanceJobs = s.ListPluginJobs("lance_vector_index", "", 5)
+	return data
+}

weed/admin/dash/vector_status_test.go

@@ -0,0 +1,49 @@
+package dash
+
+import (
+	"context"
+	"net"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/health"
+	"google.golang.org/grpc/health/grpc_health_v1"
+)
+
+func TestVectorNodeStatus(t *testing.T) {
+	for _, want := range []string{"Ready", "Not ready", "Registered (health unavailable)", "Unknown service"} {
+		t.Run(want, func(t *testing.T) {
+			listener, err := net.Listen("tcp", "127.0.0.1:0")
+			if err != nil {
+				t.Fatal(err)
+			}
+			server := grpc.NewServer()
+			if want != "Registered (health unavailable)" {
+				checker := health.NewServer()
+				state := grpc_health_v1.HealthCheckResponse_NOT_SERVING
+				if want == "Ready" {
+					state = grpc_health_v1.HealthCheckResponse_SERVING
+				}
+				if want != "Unknown service" {
+					checker.SetServingStatus("vector_pb.VectorService", state)
+				}
+				grpc_health_v1.RegisterHealthServer(server, checker)
+			}
+			go server.Serve(listener)
+			t.Cleanup(server.Stop)
+			expected := want
+			if want == "Unknown service" {
+				expected = "Registered (health unavailable)"
+			}
+			if got := vectorNodeStatus(t.Context(), listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())); got != expected {
+				t.Fatalf("got %s want %s", got, want)
+			}
+		})
+	}
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	if got := vectorNodeStatus(ctx, "127.0.0.1:1", grpc.WithTransportCredentials(insecure.NewCredentials())); got != "Unreachable" {
+		t.Fatal(got)
+	}
+}

weed/admin/handlers/admin_handlers.go

@@ -256,6 +256,7 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
 	s3TablesApi.Handle("/tags", wrapWrite(h.adminServer.UntagS3TablesResource)).Methods(http.MethodDelete)
 
 	s3VectorsApi := api.PathPrefix("/s3vectors").Subrouter()
+	s3VectorsApi.HandleFunc("/operations", h.adminServer.ExecuteS3VectorsOperationAPI).Methods(http.MethodPost)
 	s3VectorsApi.HandleFunc("/buckets", h.adminServer.ListS3VectorsBucketsAPI).Methods(http.MethodGet)
 	s3VectorsApi.Handle("/buckets", wrapWrite(h.adminServer.CreateS3VectorsBucket)).Methods(http.MethodPost)
 	s3VectorsApi.Handle("/buckets", wrapWrite(h.adminServer.DeleteS3VectorsBucket)).Methods(http.MethodDelete)

weed/admin/view/app/policy_action_suggestions.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
 )
 
 // PolicyActionSuggestions feeds the <datalist> used by the structured policy
@@ -26,7 +27,7 @@ var S3TablesPolicyActionSuggestions = func() []string {
 }()
 
 func buildPolicyActionSuggestions() []string {
-	return []string{
+	suggestions := []string{
 		// s3: actions, sourced from the constants used elsewhere for policy
 		// evaluation (weed/s3api/s3_constants/s3_action_strings.go) so the
 		// suggestion list can't drift from the strings the engine actually
@@ -120,4 +121,10 @@ func buildPolicyActionSuggestions() []string {
 		s3_constants.S3TABLES_ACTION_LIST_TAGS_FOR_RESOURCE,
 		s3_constants.S3TABLES_ACTION_UNTAG_RESOURCE,
 	}
+	suggestions = append(suggestions, "s3vectors:*")
+	for _, operation := range s3vectors.Operations() {
+		suggestions = append(suggestions, "s3vectors:"+operation)
+	}
+	return suggestions
+
 }

weed/admin/view/app/policy_action_suggestions_test.go

@@ -23,7 +23,7 @@ func TestPolicyActionSuggestions_NoDuplicates(t *testing.T) {
 
 func TestPolicyActionSuggestions_AllPrefixed(t *testing.T) {
 	for _, action := range PolicyActionSuggestions {
-		if !strings.HasPrefix(action, "s3:") && !strings.HasPrefix(action, "s3tables:") {
+		if !strings.HasPrefix(action, "s3:") && !strings.HasPrefix(action, "s3tables:") && !strings.HasPrefix(action, "s3vectors:") {
 			t.Errorf("action suggestion %q is not prefixed with a known service", action)
 		}
 	}
@@ -36,3 +36,17 @@ func TestPolicyActionSuggestions_NoEmptyStrings(t *testing.T) {
 		}
 	}
 }
+
+func TestVectorPolicySuggestions(t *testing.T) {
+	for _, want := range []string{"s3vectors:*", "s3vectors:QueryVectors", "s3vectors:CreateIndex", "s3vectors:TagResource"} {
+		found := false
+		for _, value := range PolicyActionSuggestions {
+			if value == want {
+				found = true
+			}
+		}
+		if !found {
+			t.Errorf("missing %s", want)
+		}
+	}
+}

weed/admin/view/app/s3vectors_bucket_details.templ

@@ -23,6 +23,8 @@ templ S3VectorsBucketDetails(data dash.S3VectorsBucketData) {
 			</nav>
 		</div>
 	</div>
+ @VectorStatus(data.Operations)
+ @VectorTools(data.Bucket.Name,data.Bucket.ARN,data.Policy)
 	<div class="card shadow mb-4">
 		<div class="card-body">
 			<dl class="row mb-0">
@@ -92,7 +94,7 @@ templ S3VectorsBucketDetails(data dash.S3VectorsBucketData) {
 						if len(data.Indexes) == 0 {
 							<tr>
 								<td colspan="7" class="text-center text-muted py-4">
-									No indexes. Create one with an S3 Vectors client, e.g. <code>aws s3vectors create-index</code>.
+									No indexes. Create one with the form above.
 								</td>
 							</tr>
 						}

weed/admin/view/app/s3vectors_bucket_details_templ.go

@@ -74,163 +74,162 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "</li></ol></nav></div></div><div class=\"card shadow mb-4\"><div class=\"card-body\"><dl class=\"row mb-0\"><dt class=\"col-sm-2\">ARN</dt><dd class=\"col-sm-10\"><code>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "</li></ol></nav></div></div>")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = VectorStatus(data.Operations).Render(ctx, templ_7745c5c3_Buffer)
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = VectorTools(data.Bucket.Name, data.Bucket.ARN, data.Policy).Render(ctx, templ_7745c5c3_Buffer)
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "<div class=\"card shadow mb-4\"><div class=\"card-body\"><dl class=\"row mb-0\"><dt class=\"col-sm-2\">ARN</dt><dd class=\"col-sm-10\"><code>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		var templ_7745c5c3_Var5 string
 		templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.ARN)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 30, Col: 49}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 32, Col: 49}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "</code></dd><dt class=\"col-sm-2\">Created</dt><dd class=\"col-sm-10\">")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "</code></dd><dt class=\"col-sm-2\">Created</dt><dd class=\"col-sm-10\">")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		var templ_7745c5c3_Var6 string
 		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.CreatedAt.Format("2006-01-02 15:04:05"))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 32, Col: 79}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 34, Col: 79}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "</dd><dt class=\"col-sm-2\">Encryption</dt><dd class=\"col-sm-10\">")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "</dd><dt class=\"col-sm-2\">Encryption</dt><dd class=\"col-sm-10\">")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		if data.Bucket.Encryption != "" {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<span class=\"badge bg-success\">")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "<span class=\"badge bg-success\">")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var7 string
 			templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Encryption)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 36, Col: 61}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 38, Col: 61}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "</span>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "</span>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		} else {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "<span class=\"text-muted\">None</span>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "<span class=\"text-muted\">None</span>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</dd></dl></div></div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-list-ul me-2\"></i>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "</dd></dl></div></div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-list-ul me-2\"></i>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		var templ_7745c5c3_Var8 string
 		templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d indexes", data.TotalIndexes))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 47, Col: 84}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 49, Col: 84}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, " ")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, " ")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		if data.Truncated {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "<span class=\"text-muted small ms-2\">")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "<span class=\"text-muted small ms-2\">")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var9 string
 			templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalIndexes))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 49, Col: 138}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 51, Col: 138}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "</span>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "</span>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "</h6></div><div class=\"card-body\"><div class=\"table-responsive\"><table class=\"table table-hover\" width=\"100%\" cellspacing=\"0\" id=\"s3vectorsIndexesTable\"><thead><tr><th>Name</th><th>Data Type</th><th>Dimension</th><th>Distance Metric</th><th>Non-filterable Metadata Keys</th><th>Created</th><th>Actions</th></tr></thead> <tbody>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "</h6></div><div class=\"card-body\"><div class=\"table-responsive\"><table class=\"table table-hover\" width=\"100%\" cellspacing=\"0\" id=\"s3vectorsIndexesTable\"><thead><tr><th>Name</th><th>Data Type</th><th>Dimension</th><th>Distance Metric</th><th>Non-filterable Metadata Keys</th><th>Created</th><th>Actions</th></tr></thead> <tbody>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		for _, index := range data.Indexes {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "<tr><td>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "<tr><td>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var10 string
 			templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(index.IndexName)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 71, Col: 26}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 73, Col: 26}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "<div class=\"text-muted small\">")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "<div class=\"text-muted small\">")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var11 string
 			templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(index.IndexArn)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 72, Col: 55}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 74, Col: 55}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "</div></td><td>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "</div></td><td>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var12 string
 			templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(index.DataType)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 74, Col: 28}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 76, Col: 28}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "</td><td>")
-			if templ_7745c5c3_Err != nil {
-				return templ_7745c5c3_Err
-			}
-			var templ_7745c5c3_Var13 string
-			templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", index.Dimension))
-			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 75, Col: 48}
-			}
-			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
-			if templ_7745c5c3_Err != nil {
-				return templ_7745c5c3_Err
-			}
 			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "</td><td>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			var templ_7745c5c3_Var14 string
-			templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(index.DistanceMetric)
+			var templ_7745c5c3_Var13 string
+			templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", index.Dimension))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 76, Col: 34}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 77, Col: 48}
 			}
-			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
+			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
@@ -238,109 +237,122 @@ func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
+			var templ_7745c5c3_Var14 string
+			templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(index.DistanceMetric)
+			if templ_7745c5c3_Err != nil {
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 78, Col: 34}
+			}
+			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "</td><td>")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
 			if index.MetadataConfiguration != nil && len(index.MetadataConfiguration.NonFilterableMetadataKeys) > 0 {
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "<code>")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "<code>")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 				var templ_7745c5c3_Var15 string
 				templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(strings.Join(index.MetadataConfiguration.NonFilterableMetadataKeys, ", "))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 79, Col: 91}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 81, Col: 91}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "</code>")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "</code>")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 			} else {
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "<span class=\"text-muted\">None</span>")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "<span class=\"text-muted\">None</span>")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "</td><td>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "</td><td>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var16 string
 			templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(index.CreationTime.Format("2006-01-02 15:04"))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 84, Col: 59}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 86, Col: 59}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "</td><td><button type=\"button\" class=\"btn btn-outline-danger btn-sm s3vectors-delete-index-btn\" data-index-name=\"")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "</td><td><button type=\"button\" class=\"btn btn-outline-danger btn-sm s3vectors-delete-index-btn\" data-index-name=\"")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var17 string
 			templ_7745c5c3_Var17, templ_7745c5c3_Err = templruntime.NewAttributeExpression(index.IndexName).Resolve(ctx)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 86, Col: 129}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 88, Col: 129}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var17)
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "\" title=\"Delete\"><i class=\"bi bi-trash\"></i></button></td></tr>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "\" title=\"Delete\"><i class=\"bi bi-trash\"></i></button></td></tr>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
 		if len(data.Indexes) == 0 {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "<tr><td colspan=\"7\" class=\"text-center text-muted py-4\">No indexes. Create one with an S3 Vectors client, e.g. <code>aws s3vectors create-index</code>.</td></tr>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "<tr><td colspan=\"7\" class=\"text-center text-muted py-4\">No indexes. Create one with the form above.</td></tr>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "</tbody></table></div></div></div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-shield me-2\"></i>Bucket Policy</h6></div><div class=\"card-body\">")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "</tbody></table></div></div></div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-shield me-2\"></i>Bucket Policy</h6></div><div class=\"card-body\">")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		if data.Policy != "" {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "<pre class=\"bg-light p-3 border rounded mb-0\" id=\"s3vectorsBucketPolicy\">")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "<pre class=\"bg-light p-3 border rounded mb-0\" id=\"s3vectorsBucketPolicy\">")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var18 string
 			templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(indentJSON(data.Policy))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 112, Col: 102}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 114, Col: 102}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "</pre>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "</pre>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		} else {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "<span class=\"text-muted\">No policy. Set one with <code>PutVectorBucketPolicy</code>.</span>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "<span class=\"text-muted\">No policy. Set one with <code>PutVectorBucketPolicy</code>.</span>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "</div></div><div class=\"modal fade\" id=\"deleteS3VectorsIndexModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3VectorsIndexModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3VectorsIndexModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i>Delete Index</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Delete the index <strong id=\"deleteS3VectorsIndexName\"></strong> and all its vectors?</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" id=\"deleteS3VectorsIndexBtn\"><i class=\"bi bi-trash me-1\"></i>Delete</button></div></div></div></div><div id=\"s3vectorsBucketDetails\" data-bucket-name=\"")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "</div></div><div class=\"modal fade\" id=\"deleteS3VectorsIndexModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3VectorsIndexModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3VectorsIndexModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i>Delete Index</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Delete the index <strong id=\"deleteS3VectorsIndexName\"></strong> and all its vectors?</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" id=\"deleteS3VectorsIndexBtn\"><i class=\"bi bi-trash me-1\"></i>Delete</button></div></div></div></div><div id=\"s3vectorsBucketDetails\" data-bucket-name=\"")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		var templ_7745c5c3_Var19 string
 		templ_7745c5c3_Var19, templ_7745c5c3_Err = templruntime.NewAttributeExpression(data.Bucket.Name).Resolve(ctx)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 139, Col: 69}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 141, Col: 69}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var19)
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "\"></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function () {\n\t\t\tconst bucket = document.getElementById('s3vectorsBucketDetails').dataset.bucketName;\n\t\t\tconst modalEl = document.getElementById('deleteS3VectorsIndexModal');\n\t\t\tconst modal = new bootstrap.Modal(modalEl);\n\t\t\tdocument.querySelectorAll('.s3vectors-delete-index-btn').forEach(button => {\n\t\t\t\tbutton.addEventListener('click', function () {\n\t\t\t\t\tmodalEl.dataset.indexName = this.dataset.indexName || '';\n\t\t\t\t\tdocument.getElementById('deleteS3VectorsIndexName').textContent = this.dataset.indexName || '';\n\t\t\t\t\tmodal.show();\n\t\t\t\t});\n\t\t\t});\n\t\t\tdocument.getElementById('deleteS3VectorsIndexBtn').addEventListener('click', async function () {\n\t\t\t\tconst index = modalEl.dataset.indexName;\n\t\t\t\tif (!bucket || !index) return;\n\t\t\t\ttry {\n\t\t\t\t\tconst response = await fetch(s3tBasePath('/api/s3vectors/indexes?bucket=' + encodeURIComponent(bucket) + '&index=' + encodeURIComponent(index)), { method: 'DELETE', headers: s3tWriteHeaders() });\n\t\t\t\t\tconst data = await response.json();\n\t\t\t\t\tif (!response.ok) {\n\t\t\t\t\t\talert(data.error || 'Failed to delete index');\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\talert('Failed to delete index: ' + error.message);\n\t\t\t\t}\n\t\t\t});\n\t\t});\n\t</script>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "\"></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function () {\n\t\t\tconst bucket = document.getElementById('s3vectorsBucketDetails').dataset.bucketName;\n\t\t\tconst modalEl = document.getElementById('deleteS3VectorsIndexModal');\n\t\t\tconst modal = new bootstrap.Modal(modalEl);\n\t\t\tdocument.querySelectorAll('.s3vectors-delete-index-btn').forEach(button => {\n\t\t\t\tbutton.addEventListener('click', function () {\n\t\t\t\t\tmodalEl.dataset.indexName = this.dataset.indexName || '';\n\t\t\t\t\tdocument.getElementById('deleteS3VectorsIndexName').textContent = this.dataset.indexName || '';\n\t\t\t\t\tmodal.show();\n\t\t\t\t});\n\t\t\t});\n\t\t\tdocument.getElementById('deleteS3VectorsIndexBtn').addEventListener('click', async function () {\n\t\t\t\tconst index = modalEl.dataset.indexName;\n\t\t\t\tif (!bucket || !index) return;\n\t\t\t\ttry {\n\t\t\t\t\tconst response = await fetch(s3tBasePath('/api/s3vectors/indexes?bucket=' + encodeURIComponent(bucket) + '&index=' + encodeURIComponent(index)), { method: 'DELETE', headers: s3tWriteHeaders() });\n\t\t\t\t\tconst data = await response.json();\n\t\t\t\t\tif (!response.ok) {\n\t\t\t\t\t\talert(data.error || 'Failed to delete index');\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\talert('Failed to delete index: ' + error.message);\n\t\t\t\t}\n\t\t\t});\n\t\t});\n\t</script>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}

weed/admin/view/app/s3vectors_buckets.templ

@@ -23,8 +23,9 @@ templ S3VectorsBuckets(data dash.S3VectorsBucketsData) {
 	<div class="alert alert-info mb-4">
 		<i class="bi bi-info-circle me-2"></i>
 		Vector buckets serve the Amazon S3 Vectors API on the S3 port. Each one is a LANCE table bucket that keeps its indexes in its <code>vector</code> namespace.
-		Indexes are created and filled with an S3 Vectors client, which needs a running <code>weed-vector</code> engine.
+		Create indexes and manage vectors from a bucket page or an S3 Vectors client.
 	</div>
+ @VectorStatus(data.Operations)
 	<div class="card shadow mb-4">
 		<div class="card-header py-3">
 			<h6 class="m-0 font-weight-bold text-primary">
@@ -104,9 +105,10 @@ templ S3VectorsBuckets(data dash.S3VectorsBucketsData) {
 							<input type="text" class="form-control" id="s3vectorsBucketName" name="name" placeholder="vector-bucket-name" required/>
 							<div class="form-text">3 to 63 lowercase letters, digits and hyphens, starting and ending with a letter or digit.</div>
 						</div>
-						<div class="form-text">
-							The bucket is created without encryption. To encrypt it, create it with an S3 Vectors client and <code>encryptionConfiguration</code>, which the S3 gateway checks against its SSE-S3 key.
-						</div>
+<label for="s3vectorsEncryption" class="form-label">Encryption</label>
+ <select id="s3vectorsEncryption" class="form-select"><option value="">None</option><option value="AES256">SSE-S3 (AES256)</option></select>
+ <div class="form-text">SSE-S3 requires the encryption key on the S3 gateway and vector engines.</div>
+
 					</div>
 					<div class="modal-footer">
 						<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
@@ -172,7 +174,7 @@ templ S3VectorsBuckets(data dash.S3VectorsBucketsData) {
 					const response = await fetch(s3tBasePath('/api/s3vectors/buckets'), {
 						method: 'POST',
 						headers: s3tWriteHeaders({ 'Content-Type': 'application/json' }),
-						body: JSON.stringify({ name: name })
+						body: JSON.stringify({ name, encryptionConfiguration: document.getElementById('s3vectorsEncryption').value ? {sseType:'AES256'} : undefined })
 					});
 					const data = await response.json();
 					if (!response.ok) {

weed/admin/view/app/s3vectors_buckets_templ.go

@@ -33,165 +33,173 @@ func S3VectorsBuckets(data dash.S3VectorsBucketsData) templ.Component {
 		}
 		ctx = templ.InitializeContext(ctx)
 		ctx = templ.ClearChildren(ctx)
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom\"><h1 class=\"h2\"><i class=\"bi bi-bounding-box-circles me-2\"></i>Vector Buckets</h1><div class=\"btn-toolbar mb-2 mb-md-0\"><div class=\"btn-group me-2\"><button type=\"button\" class=\"btn btn-sm btn-primary\" data-bs-toggle=\"modal\" data-bs-target=\"#createS3VectorsBucketModal\"><i class=\"bi bi-plus me-1\"></i>Create Bucket</button></div></div></div><div class=\"alert alert-info mb-4\"><i class=\"bi bi-info-circle me-2\"></i> Vector buckets serve the Amazon S3 Vectors API on the S3 port. Each one is a LANCE table bucket that keeps its indexes in its <code>vector</code> namespace. Indexes are created and filled with an S3 Vectors client, which needs a running <code>weed-vector</code> engine.</div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-bounding-box-circles me-2\"></i>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom\"><h1 class=\"h2\"><i class=\"bi bi-bounding-box-circles me-2\"></i>Vector Buckets</h1><div class=\"btn-toolbar mb-2 mb-md-0\"><div class=\"btn-group me-2\"><button type=\"button\" class=\"btn btn-sm btn-primary\" data-bs-toggle=\"modal\" data-bs-target=\"#createS3VectorsBucketModal\"><i class=\"bi bi-plus me-1\"></i>Create Bucket</button></div></div></div><div class=\"alert alert-info mb-4\"><i class=\"bi bi-info-circle me-2\"></i> Vector buckets serve the Amazon S3 Vectors API on the S3 port. Each one is a LANCE table bucket that keeps its indexes in its <code>vector</code> namespace. Create indexes and manage vectors from a bucket page or an S3 Vectors client.</div>")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = VectorStatus(data.Operations).Render(ctx, templ_7745c5c3_Buffer)
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "<div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-bounding-box-circles me-2\"></i>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		var templ_7745c5c3_Var2 string
 		templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d vector buckets", data.TotalBuckets))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 31, Col: 104}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 32, Col: 104}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, " ")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, " ")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		if data.Truncated {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "<span class=\"text-muted small ms-2\">")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "<span class=\"text-muted small ms-2\">")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var3 string
 			templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalBuckets))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 33, Col: 138}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 34, Col: 138}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "</span>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "</span>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "</h6></div><div class=\"card-body\"><div class=\"table-responsive\"><table class=\"table table-hover\" width=\"100%\" cellspacing=\"0\" id=\"s3vectorsBucketsTable\"><thead><tr><th>Name</th><th>ARN</th><th>Encryption</th><th>Created</th><th>Actions</th></tr></thead> <tbody>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "</h6></div><div class=\"card-body\"><div class=\"table-responsive\"><table class=\"table table-hover\" width=\"100%\" cellspacing=\"0\" id=\"s3vectorsBucketsTable\"><thead><tr><th>Name</th><th>ARN</th><th>Encryption</th><th>Created</th><th>Actions</th></tr></thead> <tbody>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		for _, bucket := range data.Buckets {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<tr><td><a href=\"")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<tr><td><a href=\"")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var4 string
 			templ_7745c5c3_Var4, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, "/object-store/vector-buckets/"+url.PathEscape(bucket.Name))).ResolveURL(ctx)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 53, Col: 94}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 54, Col: 94}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var4)
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "\">")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "\">")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var5 string
 			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 53, Col: 110}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 54, Col: 110}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "</a></td><td class=\"text-muted small\">")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "</a></td><td class=\"text-muted small\">")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var6 string
 			templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 55, Col: 49}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 56, Col: 49}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "</td><td>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</td><td>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			if bucket.Encryption != "" {
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "<span class=\"badge bg-success\">")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "<span class=\"badge bg-success\">")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 				var templ_7745c5c3_Var7 string
 				templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Encryption)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 58, Col: 60}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 59, Col: 60}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "</span>")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "</span>")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 			} else {
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "<span class=\"text-muted\">None</span>")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "<span class=\"text-muted\">None</span>")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "</td><td>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "</td><td>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var8 string
 			templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04"))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 63, Col: 57}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 64, Col: 57}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "</td><td><div class=\"btn-group btn-group-sm\" role=\"group\"><a class=\"btn btn-outline-primary btn-sm\" href=\"")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "</td><td><div class=\"btn-group btn-group-sm\" role=\"group\"><a class=\"btn btn-outline-primary btn-sm\" href=\"")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var9 string
 			templ_7745c5c3_Var9, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, "/object-store/vector-buckets/"+url.PathEscape(bucket.Name))).ResolveURL(ctx)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 66, Col: 134}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 67, Col: 134}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var9)
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "\" title=\"Indexes\"><i class=\"bi bi-folder2-open\"></i></a> <button type=\"button\" class=\"btn btn-outline-danger btn-sm s3vectors-delete-bucket-btn\" data-bucket-name=\"")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "\" title=\"Indexes\"><i class=\"bi bi-folder2-open\"></i></a> <button type=\"button\" class=\"btn btn-outline-danger btn-sm s3vectors-delete-bucket-btn\" data-bucket-name=\"")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var10 string
 			templ_7745c5c3_Var10, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.Name).Resolve(ctx)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 69, Col: 128}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 70, Col: 128}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var10)
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "\" title=\"Delete\"><i class=\"bi bi-trash\"></i></button></div></td></tr>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "\" title=\"Delete\"><i class=\"bi bi-trash\"></i></button></div></td></tr>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
 		if len(data.Buckets) == 0 {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "<tr><td colspan=\"5\" class=\"text-center text-muted py-4\"><i class=\"bi bi-bounding-box-circles icon-3x mb-3 text-muted\"></i><div><h5>No vector buckets found</h5></div></td></tr>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "<tr><td colspan=\"5\" class=\"text-center text-muted py-4\"><i class=\"bi bi-bounding-box-circles icon-3x mb-3 text-muted\"></i><div><h5>No vector buckets found</h5></div></td></tr>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "</tbody></table></div></div></div><div class=\"modal fade\" id=\"createS3VectorsBucketModal\" tabindex=\"-1\" aria-labelledby=\"createS3VectorsBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"createS3VectorsBucketModalLabel\"><i class=\"bi bi-plus me-2\"></i>Create Vector Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"createS3VectorsBucketForm\"><div class=\"modal-body\"><div class=\"mb-3\"><label for=\"s3vectorsBucketName\" class=\"form-label\">Bucket Name</label> <input type=\"text\" class=\"form-control\" id=\"s3vectorsBucketName\" name=\"name\" placeholder=\"vector-bucket-name\" required><div class=\"form-text\">3 to 63 lowercase letters, digits and hyphens, starting and ending with a letter or digit.</div></div><div class=\"form-text\">The bucket is created without encryption. To encrypt it, create it with an S3 Vectors client and <code>encryptionConfiguration</code>, which the S3 gateway checks against its SSE-S3 key.</div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"bi bi-plus me-1\"></i>Create</button></div></form></div></div></div><div class=\"modal fade\" id=\"deleteS3VectorsBucketModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3VectorsBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3VectorsBucketModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i>Delete Vector Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Delete the vector bucket <strong id=\"deleteS3VectorsBucketName\"></strong>? A bucket that still has indexes is not deleted.</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" id=\"deleteS3VectorsBucketBtn\"><i class=\"bi bi-trash me-1\"></i>Delete</button></div></div></div></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function () {\n\t\t\tconst deleteModalEl = document.getElementById('deleteS3VectorsBucketModal');\n\t\t\tconst deleteModal = new bootstrap.Modal(deleteModalEl);\n\t\t\tdocument.querySelectorAll('.s3vectors-delete-bucket-btn').forEach(button => {\n\t\t\t\tbutton.addEventListener('click', function () {\n\t\t\t\t\tdeleteModalEl.dataset.bucketName = this.dataset.bucketName || '';\n\t\t\t\t\tdocument.getElementById('deleteS3VectorsBucketName').textContent = this.dataset.bucketName || '';\n\t\t\t\t\tdeleteModal.show();\n\t\t\t\t});\n\t\t\t});\n\t\t\tdocument.getElementById('deleteS3VectorsBucketBtn').addEventListener('click', async function () {\n\t\t\t\tconst bucket = deleteModalEl.dataset.bucketName;\n\t\t\t\tif (!bucket) return;\n\t\t\t\ttry {\n\t\t\t\t\tconst response = await fetch(s3tBasePath('/api/s3vectors/buckets?bucket=' + encodeURIComponent(bucket)), { method: 'DELETE', headers: s3tWriteHeaders() });\n\t\t\t\t\tconst data = await response.json();\n\t\t\t\t\tif (!response.ok) {\n\t\t\t\t\t\talert(data.error || 'Failed to delete vector bucket');\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\talert('Failed to delete vector bucket: ' + error.message);\n\t\t\t\t}\n\t\t\t});\n\t\t\tdocument.getElementById('createS3VectorsBucketForm').addEventListener('submit', async function (e) {\n\t\t\t\te.preventDefault();\n\t\t\t\tconst name = document.getElementById('s3vectorsBucketName').value.trim();\n\t\t\t\ttry {\n\t\t\t\t\tconst response = await fetch(s3tBasePath('/api/s3vectors/buckets'), {\n\t\t\t\t\t\tmethod: 'POST',\n\t\t\t\t\t\theaders: s3tWriteHeaders({ 'Content-Type': 'application/json' }),\n\t\t\t\t\t\tbody: JSON.stringify({ name: name })\n\t\t\t\t\t});\n\t\t\t\t\tconst data = await response.json();\n\t\t\t\t\tif (!response.ok) {\n\t\t\t\t\t\talert(data.error || 'Failed to create vector bucket');\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\talert('Failed to create vector bucket: ' + error.message);\n\t\t\t\t}\n\t\t\t});\n\t\t});\n\t</script>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "</tbody></table></div></div></div><div class=\"modal fade\" id=\"createS3VectorsBucketModal\" tabindex=\"-1\" aria-labelledby=\"createS3VectorsBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"createS3VectorsBucketModalLabel\"><i class=\"bi bi-plus me-2\"></i>Create Vector Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"createS3VectorsBucketForm\"><div class=\"modal-body\"><div class=\"mb-3\"><label for=\"s3vectorsBucketName\" class=\"form-label\">Bucket Name</label> <input type=\"text\" class=\"form-control\" id=\"s3vectorsBucketName\" name=\"name\" placeholder=\"vector-bucket-name\" required><div class=\"form-text\">3 to 63 lowercase letters, digits and hyphens, starting and ending with a letter or digit.</div></div><label for=\"s3vectorsEncryption\" class=\"form-label\">Encryption</label> <select id=\"s3vectorsEncryption\" class=\"form-select\"><option value=\"\">None</option><option value=\"AES256\">SSE-S3 (AES256)</option></select><div class=\"form-text\">SSE-S3 requires the encryption key on the S3 gateway and vector engines.</div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"bi bi-plus me-1\"></i>Create</button></div></form></div></div></div><div class=\"modal fade\" id=\"deleteS3VectorsBucketModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3VectorsBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3VectorsBucketModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i>Delete Vector Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Delete the vector bucket <strong id=\"deleteS3VectorsBucketName\"></strong>? A bucket that still has indexes is not deleted.</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" id=\"deleteS3VectorsBucketBtn\"><i class=\"bi bi-trash me-1\"></i>Delete</button></div></div></div></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function () {\n\t\t\tconst deleteModalEl = document.getElementById('deleteS3VectorsBucketModal');\n\t\t\tconst deleteModal = new bootstrap.Modal(deleteModalEl);\n\t\t\tdocument.querySelectorAll('.s3vectors-delete-bucket-btn').forEach(button => {\n\t\t\t\tbutton.addEventListener('click', function () {\n\t\t\t\t\tdeleteModalEl.dataset.bucketName = this.dataset.bucketName || '';\n\t\t\t\t\tdocument.getElementById('deleteS3VectorsBucketName').textContent = this.dataset.bucketName || '';\n\t\t\t\t\tdeleteModal.show();\n\t\t\t\t});\n\t\t\t});\n\t\t\tdocument.getElementById('deleteS3VectorsBucketBtn').addEventListener('click', async function () {\n\t\t\t\tconst bucket = deleteModalEl.dataset.bucketName;\n\t\t\t\tif (!bucket) return;\n\t\t\t\ttry {\n\t\t\t\t\tconst response = await fetch(s3tBasePath('/api/s3vectors/buckets?bucket=' + encodeURIComponent(bucket)), { method: 'DELETE', headers: s3tWriteHeaders() });\n\t\t\t\t\tconst data = await response.json();\n\t\t\t\t\tif (!response.ok) {\n\t\t\t\t\t\talert(data.error || 'Failed to delete vector bucket');\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\talert('Failed to delete vector bucket: ' + error.message);\n\t\t\t\t}\n\t\t\t});\n\t\t\tdocument.getElementById('createS3VectorsBucketForm').addEventListener('submit', async function (e) {\n\t\t\t\te.preventDefault();\n\t\t\t\tconst name = document.getElementById('s3vectorsBucketName').value.trim();\n\t\t\t\ttry {\n\t\t\t\t\tconst response = await fetch(s3tBasePath('/api/s3vectors/buckets'), {\n\t\t\t\t\t\tmethod: 'POST',\n\t\t\t\t\t\theaders: s3tWriteHeaders({ 'Content-Type': 'application/json' }),\n\t\t\t\t\t\tbody: JSON.stringify({ name, encryptionConfiguration: document.getElementById('s3vectorsEncryption').value ? {sseType:'AES256'} : undefined })\n\t\t\t\t\t});\n\t\t\t\t\tconst data = await response.json();\n\t\t\t\t\tif (!response.ok) {\n\t\t\t\t\t\talert(data.error || 'Failed to create vector bucket');\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\talert('Failed to create vector bucket: ' + error.message);\n\t\t\t\t}\n\t\t\t});\n\t\t});\n\t</script>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}

weed/admin/view/app/s3vectors_buckets_test.go

@@ -78,3 +78,21 @@ func TestS3TablesBucketsLinksVectorBuckets(t *testing.T) {
 	require.Contains(t, page, `href="/object-store/vector-buckets/vectors"`)
 	require.NotContains(t, page, `href="/object-store/vector-buckets/warehouse"`)
 }
+
+func TestVectorOperationalStates(t *testing.T) {
+	for _, data := range []dash.VectorOperationsData{
+		{}, {DiscoveryError: "master unavailable"},
+		{Engines: []dash.VectorNode{{Address: "engine:8090", Status: "Ready"}}, MaintenanceWorkers: 1},
+	} {
+		var out bytes.Buffer
+		require.NoError(t, VectorStatus(data).Render(t.Context(), &out))
+		if data.DiscoveryError != "" {
+			require.Contains(t, out.String(), "Engine discovery failed")
+		} else if len(data.Engines) == 0 {
+			require.Contains(t, out.String(), "No vector engine is registered")
+		} else {
+			require.Contains(t, out.String(), "engine:8090")
+			require.Contains(t, out.String(), "Ready")
+		}
+	}
+}

weed/admin/view/app/vector_status.templ

@@ -0,0 +1,38 @@
+package app
+
+import (
+ "fmt"
+ "github.com/seaweedfs/seaweedfs/weed/admin/dash"
+)
+
+templ VectorStatus(data dash.VectorOperationsData) {
+ <div class="card shadow mb-4">
+  <div class="card-header">Vector Engines and Maintenance</div>
+  <div class="card-body">
+   if data.DiscoveryError != "" {
+    <div class="alert alert-warning">Engine discovery failed: { data.DiscoveryError }</div>
+   } else if len(data.Engines)==0 {
+    <div class="alert alert-warning">No vector engine is registered. Index creation, ingestion and search need a running engine.</div>
+   } else {
+    <table class="table table-sm"><thead><tr><th>Engine</th><th>Version</th><th>Status</th></tr></thead><tbody>
+     for _,engine:=range data.Engines {
+      <tr><td>{ engine.Address }</td><td>{ engine.Version }</td><td>{ engine.Status }</td></tr>
+     }
+    </tbody></table>
+   }
+   if data.MaintenanceWorkers == 0 {
+    <p class="text-warning">No vector index maintenance worker is connected. Searches still work; enable a Lance worker to build and refresh ANN indexes as data grows.</p>
+   } else {
+    <p>{ fmt.Sprintf("%d vector maintenance workers connected",data.MaintenanceWorkers) }</p>
+   }
+   <a href={ dash.PUrl(ctx,"/plugin/monitoring") }>Maintenance jobs</a>
+   if len(data.MaintenanceJobs)>0 {
+    <table class="table table-sm mt-2"><thead><tr><th>Recent job</th><th>State</th><th>Result</th></tr></thead><tbody>
+     for _,job:=range data.MaintenanceJobs {
+      <tr><td>{ job.JobID }</td><td>{ job.State }</td><td>{ job.ResultSummary } { job.ErrorMessage }</td></tr>
+     }
+    </tbody></table>
+   }
+  </div>
+ </div>
+}

weed/admin/view/app/vector_status_templ.go

@@ -0,0 +1,233 @@
+// Code generated by templ - DO NOT EDIT.
+
+// templ: version: v0.3.1100
+
+package app
+
+//lint:file-ignore SA4006 This context is only used if a nested component is present.
+
+import "github.com/a-h/templ"
+import templruntime "github.com/a-h/templ/runtime"
+
+import (
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
+)
+
+func VectorStatus(data dash.VectorOperationsData) templ.Component {
+	return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
+		templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
+		if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
+			return templ_7745c5c3_CtxErr
+		}
+		templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
+		if !templ_7745c5c3_IsBuffer {
+			defer func() {
+				templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
+				if templ_7745c5c3_Err == nil {
+					templ_7745c5c3_Err = templ_7745c5c3_BufErr
+				}
+			}()
+		}
+		ctx = templ.InitializeContext(ctx)
+		ctx = templ.ClearChildren(ctx)
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"card shadow mb-4\"><div class=\"card-header\">Vector Engines and Maintenance</div><div class=\"card-body\">")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		if data.DiscoveryError != "" {
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "<div class=\"alert alert-warning\">Engine discovery failed: ")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			var templ_7745c5c3_Var2 string
+			templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(data.DiscoveryError)
+			if templ_7745c5c3_Err != nil {
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 13, Col: 83}
+			}
+			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "</div>")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+		} else if len(data.Engines) == 0 {
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "<div class=\"alert alert-warning\">No vector engine is registered. Index creation, ingestion and search need a running engine.</div>")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+		} else {
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "<table class=\"table table-sm\"><thead><tr><th>Engine</th><th>Version</th><th>Status</th></tr></thead><tbody>")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			for _, engine := range data.Engines {
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<tr><td>")
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				var templ_7745c5c3_Var3 string
+				templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(engine.Address)
+				if templ_7745c5c3_Err != nil {
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 19, Col: 30}
+				}
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "</td><td>")
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				var templ_7745c5c3_Var4 string
+				templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(engine.Version)
+				if templ_7745c5c3_Err != nil {
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 19, Col: 57}
+				}
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "</td><td>")
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				var templ_7745c5c3_Var5 string
+				templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(engine.Status)
+				if templ_7745c5c3_Err != nil {
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 19, Col: 83}
+				}
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "</td></tr>")
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+			}
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</tbody></table>")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+		}
+		if data.MaintenanceWorkers == 0 {
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "<p class=\"text-warning\">No vector index maintenance worker is connected. Searches still work; enable a Lance worker to build and refresh ANN indexes as data grows.</p>")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+		} else {
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "<p>")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			var templ_7745c5c3_Var6 string
+			templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d vector maintenance workers connected", data.MaintenanceWorkers))
+			if templ_7745c5c3_Err != nil {
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 26, Col: 87}
+			}
+			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "</p>")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "<a href=\"")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		var templ_7745c5c3_Var7 string
+		templ_7745c5c3_Var7, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, "/plugin/monitoring")).ResolveURL(ctx)
+		if templ_7745c5c3_Err != nil {
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 28, Col: 48}
+		}
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var7)
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "\">Maintenance jobs</a> ")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		if len(data.MaintenanceJobs) > 0 {
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "<table class=\"table table-sm mt-2\"><thead><tr><th>Recent job</th><th>State</th><th>Result</th></tr></thead><tbody>")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			for _, job := range data.MaintenanceJobs {
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "<tr><td>")
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				var templ_7745c5c3_Var8 string
+				templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(job.JobID)
+				if templ_7745c5c3_Err != nil {
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 32, Col: 25}
+				}
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "</td><td>")
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				var templ_7745c5c3_Var9 string
+				templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(job.State)
+				if templ_7745c5c3_Err != nil {
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 32, Col: 47}
+				}
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "</td><td>")
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				var templ_7745c5c3_Var10 string
+				templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(job.ResultSummary)
+				if templ_7745c5c3_Err != nil {
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 32, Col: 77}
+				}
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, " ")
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				var templ_7745c5c3_Var11 string
+				templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(job.ErrorMessage)
+				if templ_7745c5c3_Err != nil {
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/vector_status.templ`, Line: 32, Col: 98}
+				}
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "</td></tr>")
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+			}
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "</tbody></table>")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "</div></div>")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		return nil
+	})
+}
+
+var _ = templruntime.GeneratedTemplate

weed/admin/view/app/vector_tools.templ

@@ -0,0 +1,85 @@
+package app
+
+
+templ VectorTools(bucket, arn, policy string) {
+ <div class="card shadow mb-4" id="vectorTools" data-bucket={ bucket } data-arn={ arn }>
+  <div class="card-header">Manage Indexes, Policies and Vectors</div>
+  <div class="card-body">
+   <form id="vectorIndexForm" class="row g-2 mb-3">
+    <div class="col"><label for="vectorIndexName" class="form-label">Index name</label><input id="vectorIndexName" class="form-control" required/></div>
+    <div class="col"><label for="vectorDimension" class="form-label">Dimension</label><input id="vectorDimension" class="form-control" type="number" min="1" max="4096" required/></div>
+    <div class="col"><label for="vectorDistance" class="form-label">Distance</label><select id="vectorDistance" class="form-select"><option value="cosine">Cosine</option><option value="euclidean">Euclidean</option></select></div>
+    <div class="col-auto align-self-end"><button class="btn btn-primary" type="submit">Create Index</button></div>
+   </form>
+   <form id="vectorPolicyForm" class="mb-3">
+    <label for="vectorPolicy" class="form-label">Bucket policy</label><textarea id="vectorPolicy" class="form-control font-monospace" rows="5">{ policy }</textarea>
+    <button class="btn btn-outline-primary mt-2" type="submit">Save Policy</button>
+    <button class="btn btn-outline-danger mt-2" type="button" id="vectorDeletePolicy">Delete Policy</button>
+   </form>
+   <form id="vectorOperationForm">
+    <label for="vectorOperation" class="form-label">Vector and tagging operations</label>
+    <select id="vectorOperation" class="form-select mb-2">
+     <option>QueryVectors</option><option>PutVectors</option><option>GetVectors</option><option>ListVectors</option><option>DeleteVectors</option><option>ListTagsForResource</option><option>TagResource</option><option>UntagResource</option>
+    </select>
+    <label for="vectorRequest" class="form-label">Request JSON</label><textarea id="vectorRequest" class="form-control font-monospace" rows="7" required></textarea>
+    <button class="btn btn-outline-primary mt-2" type="submit">Run</button>
+   </form>
+   <pre id="vectorResult" class="mt-3 mb-0" role="status"></pre>
+  </div>
+ </div>
+ <script>
+ document.addEventListener('DOMContentLoaded', () => {
+  const root = document.getElementById('vectorTools');
+  const bucket = root.dataset.bucket;
+  const output = document.getElementById('vectorResult');
+  const run = async (operation, request) => {
+   output.textContent = 'Running…';
+   const response = await fetch(s3tBasePath('/api/s3vectors/operations'), {method:'POST', headers:s3tWriteHeaders({'Content-Type':'application/json'}), body:JSON.stringify({operation,request})});
+   const result = await response.json();
+   output.textContent = JSON.stringify(result,null,2);
+   if (!response.ok) throw new Error(result.error || 'Operation failed');
+   return result;
+  };
+  const submit = (id, action) => document.getElementById(id).addEventListener('submit', async event => {
+   event.preventDefault();
+   const buttons = event.target.querySelectorAll('button');
+   buttons.forEach(button => button.disabled = true);
+   try { await action(); } catch(error) {output.textContent=error.message;} finally {buttons.forEach(button => button.disabled = false);}
+  });
+  submit('vectorIndexForm', async () => {
+   await run('CreateIndex',{vectorBucketName:bucket,indexName:document.getElementById('vectorIndexName').value,dimension:Number(document.getElementById('vectorDimension').value),distanceMetric:document.getElementById('vectorDistance').value,dataType:'float32'});
+   location.reload();
+  });
+  submit('vectorPolicyForm', async () => {
+   const policy=document.getElementById('vectorPolicy').value;
+   JSON.parse(policy);
+   await run('PutVectorBucketPolicy',{vectorBucketName:bucket,policy});
+   location.reload();
+  });
+  document.getElementById('vectorDeletePolicy').addEventListener('click',async () => {
+   if (!confirm('Delete this bucket policy?')) return;
+   const controls=document.getElementById('vectorPolicyForm').querySelectorAll('button,textarea');
+   controls.forEach(control => control.disabled=true);
+   try {await run('DeleteVectorBucketPolicy',{vectorBucketName:bucket});location.reload();} catch(error) {output.textContent=error.message;} finally {controls.forEach(control => control.disabled=false);}
+  });
+  const operation=document.getElementById('vectorOperation');
+  const request=document.getElementById('vectorRequest');
+  const example=() => {
+   const op=operation.value;
+   let value={vectorBucketName:bucket,indexName:document.getElementById('vectorIndexName').value || 'my-index'};
+   if (op==='QueryVectors') Object.assign(value,{queryVector:{float32:[0.1,0.2,0.3]},topK:5,returnDistance:true});
+   if (op==='PutVectors') value.vectors=[{key:'example',data:{float32:[0.1,0.2,0.3]}}];
+   if (op==='GetVectors' || op==='DeleteVectors') value.keys=['example'];
+   if (op.includes('Tag') || op==='UntagResource') {
+    value={resourceArn:root.dataset.arn};
+    if (op==='TagResource') value.tags={environment:'development'};
+    if (op==='UntagResource') value.tagKeys=['environment'];
+   }
+   request.value=JSON.stringify(value,null,2);
+  };
+  operation.addEventListener('change',example);
+  example();
+  submit('vectorOperationForm',()=>run(operation.value,JSON.parse(request.value)));
+ });
+ </script>
+}

weed/admin/view/app/vector_tools_templ.go

@@ -0,0 +1,76 @@
+// Code generated by templ - DO NOT EDIT.
+
+// templ: version: v0.3.1100
+
+package app
+
+//lint:file-ignore SA4006 This context is only used if a nested component is present.
+
+import "github.com/a-h/templ"
+import templruntime "github.com/a-h/templ/runtime"
+
+func VectorTools(bucket, arn, policy string) templ.Component {
+	return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
+		templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
+		if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
+			return templ_7745c5c3_CtxErr
+		}
+		templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
+		if !templ_7745c5c3_IsBuffer {
+			defer func() {
+				templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
+				if templ_7745c5c3_Err == nil {
+					templ_7745c5c3_Err = templ_7745c5c3_BufErr
+				}
+			}()
+		}
+		ctx = templ.InitializeContext(ctx)
+		ctx = templ.ClearChildren(ctx)
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"card shadow mb-4\" id=\"vectorTools\" data-bucket=\"")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		var templ_7745c5c3_Var2 string
+		templ_7745c5c3_Var2, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket).Resolve(ctx)
+		if templ_7745c5c3_Err != nil {
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/vector_tools.templ`, Line: 5, Col: 68}
+		}
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var2)
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\" data-arn=\"")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		var templ_7745c5c3_Var3 string
+		templ_7745c5c3_Var3, templ_7745c5c3_Err = templruntime.NewAttributeExpression(arn).Resolve(ctx)
+		if templ_7745c5c3_Err != nil {
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/vector_tools.templ`, Line: 5, Col: 85}
+		}
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var3)
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "\"><div class=\"card-header\">Manage Indexes, Policies and Vectors</div><div class=\"card-body\"><form id=\"vectorIndexForm\" class=\"row g-2 mb-3\"><div class=\"col\"><label for=\"vectorIndexName\" class=\"form-label\">Index name</label><input id=\"vectorIndexName\" class=\"form-control\" required></div><div class=\"col\"><label for=\"vectorDimension\" class=\"form-label\">Dimension</label><input id=\"vectorDimension\" class=\"form-control\" type=\"number\" min=\"1\" max=\"4096\" required></div><div class=\"col\"><label for=\"vectorDistance\" class=\"form-label\">Distance</label><select id=\"vectorDistance\" class=\"form-select\"><option value=\"cosine\">Cosine</option><option value=\"euclidean\">Euclidean</option></select></div><div class=\"col-auto align-self-end\"><button class=\"btn btn-primary\" type=\"submit\">Create Index</button></div></form><form id=\"vectorPolicyForm\" class=\"mb-3\"><label for=\"vectorPolicy\" class=\"form-label\">Bucket policy</label><textarea id=\"vectorPolicy\" class=\"form-control font-monospace\" rows=\"5\">")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		var templ_7745c5c3_Var4 string
+		templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(policy)
+		if templ_7745c5c3_Err != nil {
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/vector_tools.templ`, Line: 15, Col: 151}
+		}
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "</textarea> <button class=\"btn btn-outline-primary mt-2\" type=\"submit\">Save Policy</button> <button class=\"btn btn-outline-danger mt-2\" type=\"button\" id=\"vectorDeletePolicy\">Delete Policy</button></form><form id=\"vectorOperationForm\"><label for=\"vectorOperation\" class=\"form-label\">Vector and tagging operations</label> <select id=\"vectorOperation\" class=\"form-select mb-2\"><option>QueryVectors</option><option>PutVectors</option><option>GetVectors</option><option>ListVectors</option><option>DeleteVectors</option><option>ListTagsForResource</option><option>TagResource</option><option>UntagResource</option></select> <label for=\"vectorRequest\" class=\"form-label\">Request JSON</label><textarea id=\"vectorRequest\" class=\"form-control font-monospace\" rows=\"7\" required></textarea> <button class=\"btn btn-outline-primary mt-2\" type=\"submit\">Run</button></form><pre id=\"vectorResult\" class=\"mt-3 mb-0\" role=\"status\"></pre></div></div><script>\n document.addEventListener('DOMContentLoaded', () => {\n  const root = document.getElementById('vectorTools');\n  const bucket = root.dataset.bucket;\n  const output = document.getElementById('vectorResult');\n  const run = async (operation, request) => {\n   output.textContent = 'Running…';\n   const response = await fetch(s3tBasePath('/api/s3vectors/operations'), {method:'POST', headers:s3tWriteHeaders({'Content-Type':'application/json'}), body:JSON.stringify({operation,request})});\n   const result = await response.json();\n   output.textContent = JSON.stringify(result,null,2);\n   if (!response.ok) throw new Error(result.error || 'Operation failed');\n   return result;\n  };\n  const submit = (id, action) => document.getElementById(id).addEventListener('submit', async event => {\n   event.preventDefault();\n   const buttons = event.target.querySelectorAll('button');\n   buttons.forEach(button => button.disabled = true);\n   try { await action(); } catch(error) {output.textContent=error.message;} finally {buttons.forEach(button => button.disabled = false);}\n  });\n  submit('vectorIndexForm', async () => {\n   await run('CreateIndex',{vectorBucketName:bucket,indexName:document.getElementById('vectorIndexName').value,dimension:Number(document.getElementById('vectorDimension').value),distanceMetric:document.getElementById('vectorDistance').value,dataType:'float32'});\n   location.reload();\n  });\n  submit('vectorPolicyForm', async () => {\n   const policy=document.getElementById('vectorPolicy').value;\n   JSON.parse(policy);\n   await run('PutVectorBucketPolicy',{vectorBucketName:bucket,policy});\n   location.reload();\n  });\n  document.getElementById('vectorDeletePolicy').addEventListener('click',async () => {\n   if (!confirm('Delete this bucket policy?')) return;\n   const controls=document.getElementById('vectorPolicyForm').querySelectorAll('button,textarea');\n   controls.forEach(control => control.disabled=true);\n   try {await run('DeleteVectorBucketPolicy',{vectorBucketName:bucket});location.reload();} catch(error) {output.textContent=error.message;} finally {controls.forEach(control => control.disabled=false);}\n  });\n  const operation=document.getElementById('vectorOperation');\n  const request=document.getElementById('vectorRequest');\n  const example=() => {\n   const op=operation.value;\n   let value={vectorBucketName:bucket,indexName:document.getElementById('vectorIndexName').value || 'my-index'};\n   if (op==='QueryVectors') Object.assign(value,{queryVector:{float32:[0.1,0.2,0.3]},topK:5,returnDistance:true});\n   if (op==='PutVectors') value.vectors=[{key:'example',data:{float32:[0.1,0.2,0.3]}}];\n   if (op==='GetVectors' || op==='DeleteVectors') value.keys=['example'];\n   if (op.includes('Tag') || op==='UntagResource') {\n    value={resourceArn:root.dataset.arn};\n    if (op==='TagResource') value.tags={environment:'development'};\n    if (op==='UntagResource') value.tagKeys=['environment'];\n   }\n   request.value=JSON.stringify(value,null,2);\n  };\n  operation.addEventListener('change',example);\n  example();\n  submit('vectorOperationForm',()=>run(operation.value,JSON.parse(request.value)));\n });\n </script>")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		return nil
+	})
+}
+
+var _ = templruntime.GeneratedTemplate

weed/command/admin.go

@@ -516,14 +516,6 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,
 		return fmt.Errorf("apply admin.toml to plugin config: %w", err)
 	}
 
-	// Show discovered filers
-	filers := adminServer.GetAllFilers()
-	if len(filers) > 0 {
-		glog.Infof("Discovered filers: %s", strings.Join(filers, ", "))
-	} else {
-		glog.Infof("No filers discovered from masters")
-	}
-
 	// Start worker gRPC server for worker connections. The worker gRPC binds
 	// to its own address (workerGrpcBindIp) which, unlike the HTTP ip, does
 	// not auto-upgrade to 0.0.0.0 based on adminPassword, since the worker

weed/command/mini.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"maps"
 	"math/bits"
 	"net"
 	"net/http"
@@ -415,6 +416,11 @@ var (
 	miniS3AutoCreateBucket          = cmdMini.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
 	miniBucket                      = cmdMini.Flag.String("bucket", "", "comma-separated S3 bucket names to create on startup if they do not already exist; leave empty to skip. Falls back to S3_BUCKET env var.")
 	miniTableBucket                 = cmdMini.Flag.String("tableBucket", "", "comma-separated S3 Tables buckets to create on startup if they do not already exist, each name[:FORMAT] with FORMAT one of ICEBERG (default) or LANCE, e.g. warehouse,vectors:LANCE; leave empty to skip. Falls back to S3_TABLE_BUCKET env var.")
+	miniVector                      = cmdMini.Flag.Bool("vector", false, "start the weed-vector engine for vector indexes and search")
+	miniVectorBinary                = cmdMini.Flag.String("vector.binary", "weed-vector", "path to the vector engine binary")
+	miniVectorPort                  = cmdMini.Flag.Int("vector.port", 8090, "vector engine gRPC port")
+	miniVectorMetricsPort           = cmdMini.Flag.Int("vector.metricsPort", 0, "optional vector HTTP health and metrics port")
+	miniVectorBucket                = cmdMini.Flag.String("vectorBucket", "", "comma-separated vector buckets to create; falls back to S3_VECTOR_BUCKET")
 )
 
 // getBindIp determines the bind IP address based on miniIp and miniBindIp flags
@@ -874,9 +880,9 @@ func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reserv
 	// Skip if this port is reserved for gRPC calculation
 	if reservedPorts[original] {
 		if isExplicitPort {
-			return fmt.Errorf("port %d for %s (specified by flag %s) is reserved for gRPC calculation and cannot be used", original, serviceName, flagName)
+			return fmt.Errorf("port %d for %s (specified by flag %s) is reserved by another mini service and cannot be used", original, serviceName, flagName)
 		}
-		glog.Warningf("Port %d for %s is reserved for gRPC calculation, finding alternative...", original, serviceName)
+		glog.Warningf("Port %d for %s is reserved by another mini service, finding alternative...", original, serviceName)
 		newPort := findAvailablePortOnIP(ip, original+1, 100, reservedPorts, listenOnLocalhost)
 		if newPort == 0 {
 			glog.Errorf("Could not find available port for %s starting from %d, will use original %d and fail on binding", serviceName, original+1, original)
@@ -950,22 +956,41 @@ func ensureAllPortsAvailableOnIP(bindIp string, listenOnLocalhost bool) error {
 			}{miniS3Options.portLance, "Lance", "s3.port.lance", nil, listenOnLocalhost})
 		}
 	}
-	// WebDAV and the Admin UI bind only -ip.bind; they never open a loopback
-	// companion listener, so ports held elsewhere cannot collide with them.
+	if *miniVector {
+		portConfigs = append(portConfigs, struct {
+			port      *int
+			name      string
+			flagName  string
+			grpcPtr   *int
+			localhost bool
+		}{miniVectorPort, "Vector", "vector.port", nil, false})
+		if *miniVectorMetricsPort > 0 {
+			portConfigs = append(portConfigs, struct {
+				port      *int
+				name      string
+				flagName  string
+				grpcPtr   *int
+				localhost bool
+			}{miniVectorMetricsPort, "Vector metrics", "vector.metricsPort", nil, false})
+		}
+	}
+
+	if *miniEnableWebDAV {
+		portConfigs = append(portConfigs, struct {
+			port      *int
+			name      string
+			flagName  string
+			grpcPtr   *int
+			localhost bool
+		}{miniWebDavOptions.port, "WebDAV", "webdav.port", nil, false})
+	}
 	portConfigs = append(portConfigs, struct {
 		port      *int
 		name      string
 		flagName  string
 		grpcPtr   *int
 		localhost bool
-	}{miniWebDavOptions.port, "WebDAV", "webdav.port", nil, false},
-		struct {
-			port      *int
-			name      string
-			flagName  string
-			grpcPtr   *int
-			localhost bool
-		}{miniAdminOptions.port, "Admin", "admin.port", miniAdminOptions.grpcPort, false})
+	}{miniAdminOptions.port, "Admin", "admin.port", miniAdminOptions.grpcPort, false})
 
 	// First, reserve all gRPC ports that will be calculated to prevent HTTP port allocation from using them
 	// This prevents collisions like: HTTP port moves to X, then gRPC port is calculated as Y where Y == X
@@ -978,6 +1003,16 @@ func ensureAllPortsAvailableOnIP(bindIp string, listenOnLocalhost bool) error {
 		}
 	}
 
+	chosenPorts := make(map[int]string)
+	for _, config := range portConfigs {
+		if *config.port > 0 && explicitFlags[config.flagName] {
+			if previous, exists := chosenPorts[*config.port]; exists {
+				return fmt.Errorf("port %d is shared by explicit flags %s and %s", *config.port, previous, config.flagName)
+			}
+			chosenPorts[*config.port] = config.flagName
+		}
+	}
+
 	applyMiniAdminCredentialFallback(&miniAdminOptions)
 	adminBindIP := miniAdminEffectiveBindIP(bindIp)
 
@@ -991,9 +1026,19 @@ func ensureAllPortsAvailableOnIP(bindIp string, listenOnLocalhost bool) error {
 			checkIP = adminBindIP
 		}
 		original := *config.port
-		if err := ensurePortAvailableOnIP(config.port, config.name, checkIP, reservedPorts, config.flagName, config.localhost); err != nil {
+		unavailablePorts := maps.Clone(reservedPorts)
+		for port, flag := range chosenPorts {
+			if flag != config.flagName {
+				unavailablePorts[port] = true
+			}
+		}
+
+		if err := ensurePortAvailableOnIP(config.port, config.name, checkIP, unavailablePorts, config.flagName, config.localhost); err != nil {
 			return err
 		}
+		if *config.port > 0 {
+			chosenPorts[*config.port] = config.flagName
+		}
 		// If port was changed, update the reserved gRPC ports mapping
 		if *config.port != original && config.grpcPtr != nil && *config.grpcPtr == 0 {
 			delete(reservedPorts, original+GrpcPortOffset)
@@ -1105,6 +1150,13 @@ func initializeGrpcPortsOnIP(bindIp string, listenOnLocalhost bool) map[int]bool
 		}
 	}
 
+	if *miniVector {
+		allocatedPorts[*miniVectorPort] = true
+		if *miniVectorMetricsPort > 0 {
+			allocatedPorts[*miniVectorMetricsPort] = true
+		}
+	}
+
 	grpcConfigs := []struct {
 		httpPort  *int
 		grpcPort  *int
@@ -1641,6 +1693,32 @@ func runMini(cmd *Command, args []string) bool {
 		os.Unsetenv("S3_TABLE_BUCKET")
 	}
 
+	vectorBuckets := *miniVectorBucket
+	if vectorBuckets == "" {
+		vectorBuckets = os.Getenv("S3_VECTOR_BUCKET")
+	}
+	var vectorEntries []tableBucketEntry
+	for _, name := range parseBucketList(vectorBuckets) {
+		vectorEntries = append(vectorEntries, tableBucketEntry{name: name, format: s3tables.FormatLance})
+	}
+	if len(vectorEntries) > 0 && !*miniEnableS3 {
+		glog.Errorf("vector buckets require the S3 gateway")
+		triggerMiniClientsShutdown(5 * time.Second)
+		return false
+	}
+	if ready, err := createMiniTableBuckets(vectorEntries); err != nil || len(ready) != len(vectorEntries) {
+		glog.Errorf("created %d of %d requested vector buckets: %v", len(ready), len(vectorEntries), err)
+		triggerMiniClientsShutdown(5 * time.Second)
+		return false
+	}
+	if *miniVector {
+		if err := startMiniVector(); err != nil {
+			glog.Errorf("start vector engine: %v", err)
+			triggerMiniClientsShutdown(5 * time.Second)
+			return false
+		}
+	}
+
 	// Print welcome message after all services are running
 	printWelcomeMessage()
 
@@ -2423,6 +2501,14 @@ func ensureMiniTableBuckets(buckets []tableBucketEntry) ([]tableBucketEntry, err
 		return nil, nil
 	}
 
+	return createMiniTableBuckets(servable)
+}
+
+func createMiniTableBuckets(servable []tableBucketEntry) ([]tableBucketEntry, error) {
+	if len(servable) == 0 {
+		return nil, nil
+	}
+
 	filerAddress := pb.NewServerAddress(*miniIp, *miniFilerOptions.port, *miniFilerOptions.portGrpc)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 

weed/command/mini_vector.go

@@ -0,0 +1,130 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func startMiniVector() error {
+	binary := *miniVectorBinary
+	if binary == "weed-vector" {
+		if executable, err := os.Executable(); err == nil {
+			candidate := filepath.Join(filepath.Dir(executable), binary)
+			if resolved, err := exec.LookPath(candidate); err == nil {
+				binary = resolved
+			}
+		}
+	}
+	if _, err := exec.LookPath(binary); err != nil {
+		return fmt.Errorf("%s is unavailable: install the vector component or set -vector.binary: %w", binary, err)
+	}
+	address := util.JoinHostPort(*miniIp, *miniVectorPort)
+	bindIP := *miniBindIp
+	if bindIP == "" {
+		bindIP = *miniIp
+	}
+	if net.ParseIP(bindIP) == nil {
+		ctx, cancel := context.WithTimeout(miniClientsCtx(), 5*time.Second)
+		addresses, err := net.DefaultResolver.LookupIPAddr(ctx, bindIP)
+		cancel()
+		if err != nil {
+			return fmt.Errorf("resolve vector bind address %s: %w", bindIP, err)
+		}
+		if len(addresses) == 0 {
+			return fmt.Errorf("vector bind address %s has no IP addresses", bindIP)
+		}
+		bindIP = addresses[0].String()
+	}
+
+	args := []string{"--listen=" + util.JoinHostPort(bindIP, *miniVectorPort), "--advertise=" + address, "--masters=" + string(pb.NewServerAddress(*miniIp, *miniMasterOptions.port, *miniMasterOptions.portGrpc))}
+	if miniFilerOptions.filerGroup != nil && *miniFilerOptions.filerGroup != "" {
+		args = append(args, "--filer-group="+*miniFilerOptions.filerGroup)
+	}
+	if *miniVectorMetricsPort > 0 {
+		args = append(args, fmt.Sprintf("--metrics-port=%d", *miniVectorMetricsPort), "--metrics-ip="+bindIP)
+	}
+	if directory := util.ConfigurationFileDirectory.String(); directory != "" {
+		args = append(args, "--config-dir="+util.ResolvePath(directory))
+	}
+	command := exec.CommandContext(miniClientsCtx(), binary, args...)
+	command.Stdout, command.Stderr = os.Stdout, os.Stderr
+	if err := command.Start(); err != nil {
+		return err
+	}
+	done := trackMiniClient()
+	exited := make(chan error, 1)
+	go func() {
+		defer done()
+		err := command.Wait()
+		exited <- err
+		if err != nil && miniClientsCtx().Err() == nil {
+			glog.Errorf("vector engine exited: %v", err)
+		}
+	}()
+	ctx, cancel := context.WithTimeout(miniClientsCtx(), 30*time.Second)
+	defer cancel()
+	ticker := time.NewTicker(100 * time.Millisecond)
+	defer ticker.Stop()
+	for {
+		select {
+		case err := <-exited:
+			return fmt.Errorf("vector engine exited before readiness: %v", err)
+		case <-ctx.Done():
+			_ = command.Process.Kill()
+			return ctx.Err()
+		case <-ticker.C:
+			conn, err := net.DialTimeout("tcp", miniVectorLocalAddress(bindIP, *miniVectorPort), 100*time.Millisecond)
+			if err == nil {
+				_ = conn.Close()
+				if !miniVectorRegistered(ctx, address) {
+					continue
+				}
+				glog.V(0).Infof("vector engine ready on %s", address)
+				return nil
+			}
+		}
+	}
+}
+
+func miniVectorLocalAddress(bindIP string, port int) string {
+	if ip := net.ParseIP(bindIP); ip != nil && ip.IsUnspecified() {
+		if ip.To4() != nil {
+			bindIP = "127.0.0.1"
+		} else {
+			bindIP = "::1"
+		}
+	}
+	return util.JoinHostPort(bindIP, port)
+}
+
+func miniVectorRegistered(ctx context.Context, address string) bool {
+	ctx, cancel := context.WithTimeout(ctx, time.Second)
+	defer cancel()
+	master := pb.NewServerAddress(*miniIp, *miniMasterOptions.port, *miniMasterOptions.portGrpc)
+	ready := false
+	_ = pb.WithMasterClient(ctx, false, master, security.LoadClientTLS(util.GetViper(), "grpc.client"), false, func(client master_pb.SeaweedClient) error {
+		response, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{ClientType: "vector", FilerGroup: *miniFilerOptions.filerGroup})
+		if err != nil {
+			return err
+		}
+		for _, node := range response.ClusterNodes {
+			if node.Address == address {
+				ready = true
+				break
+			}
+		}
+		return nil
+	})
+	return ready
+}

weed/command/mini_vector_test.go

@@ -0,0 +1,21 @@
+package command
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestMiniVectorRejectsExplicitListenerConflict(t *testing.T) {
+	oldVector, oldPort, oldMetrics, oldFlags := *miniVector, *miniVectorPort, *miniVectorMetricsPort, explicitFlags
+	defer func() {
+		*miniVector, *miniVectorPort, *miniVectorMetricsPort, explicitFlags = oldVector, oldPort, oldMetrics, oldFlags
+	}()
+	*miniVector = true
+	*miniVectorPort = 8090
+	*miniVectorMetricsPort = 8090
+	explicitFlags = map[string]bool{"vector.port": true, "vector.metricsPort": true}
+	err := ensureAllPortsAvailableOnIP("127.0.0.1", false)
+	if err == nil || !strings.Contains(err.Error(), "shared by explicit flags vector.port and vector.metricsPort") {
+		t.Fatalf("listener conflict not rejected: %v", err)
+	}
+}

weed/command/s3.go

@@ -10,7 +10,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"sync"
 	"sync/atomic"
 	"time"
 
@@ -20,10 +19,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/vector_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/iceberg"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/lance"
@@ -37,8 +34,6 @@ import (
 	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
 )
 
 var (
@@ -452,7 +447,7 @@ func (s3opt *S3Options) startS3Server() bool {
 	// backend is reachable — /s3vectors/ in the SigV4 credential scope picks
 	// the vectors router.
 	vectorsHandler := http.Handler(router)
-	if vectorsRouter := s3opt.vectorsRouter(s3ApiServer, grpcDialOption, masterAddresses); vectorsRouter != nil {
+	if vectorsRouter := s3opt.vectorsRouter(s3ApiServer, grpcDialOption, masterAddresses, filerGroup); vectorsRouter != nil {
 		vr, mainRouter := vectorsRouter, router
 		vectorsHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			if isVectorsRequest(r) {
@@ -739,12 +734,12 @@ func (s3opt *S3Options) startLanceServer(listenOnLocalhost bool) *http.Server {
 // vectorsRouter builds the S3 Vectors API router — workers register to the
 // master and the backend resolves them lazily, so a worker that comes up
 // after the gateway still gets picked up on first query.
-func (s3opt *S3Options) vectorsRouter(s3ApiServer *s3api.S3ApiServer, grpcDialOption grpc.DialOption, masterAddresses []pb.ServerAddress) *mux.Router {
+func (s3opt *S3Options) vectorsRouter(s3ApiServer *s3api.S3ApiServer, grpcDialOption grpc.DialOption, masterAddresses []pb.ServerAddress, filerGroup string) *mux.Router {
 	if len(masterAddresses) == 0 {
 		glog.Warningf("S3 Vectors API disabled: no weed-vector registered on master")
 		return nil
 	}
-	backend := &vectorDiscovery{masters: masterAddresses, grpcDialOption: grpcDialOption, discover: discoverVectorWorkers}
+	backend := s3vectors.NewVectorDiscovery(func() []pb.ServerAddress { return masterAddresses }, grpcDialOption, filerGroup)
 	// In-process describe — the router resolves locations and vended
 	// storage options through the catalog directly, never over HTTP. The
 	// lookup itself runs under the trusted identity: the caller's vector
@@ -769,230 +764,6 @@ func (s3opt *S3Options) vectorsRouter(s3ApiServer *s3api.S3ApiServer, grpcDialOp
 	return vectorsRouter
 }
 
-// vectorDiscovery resolves the currently registered weed-vector through the
-// master and caches the connection; workers joining after gateway startup
-// are picked up on first query.
-type vectorDiscovery struct {
-	masters        []pb.ServerAddress
-	grpcDialOption grpc.DialOption
-	discover       func([]pb.ServerAddress, grpc.DialOption) []string
-	mu             sync.Mutex
-	worker         string
-	conn           *grpc.ClientConn
-	client         vector_pb.VectorServiceClient
-	nextResolve    time.Time
-	resolving      chan struct{} // closed when the in-flight resolve lands
-}
-
-// reset drops the cached worker so the next call rediscovers — invoked on
-// transport errors only, and only when the failed client is still the
-// cached one (a concurrent request may have already re-resolved).
-func (d *vectorDiscovery) reset(c vector_pb.VectorServiceClient) {
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	if c != d.client {
-		return
-	}
-	if d.conn != nil {
-		d.conn.Close()
-	}
-	d.worker, d.conn, d.client = "", nil, nil
-	d.nextResolve = time.Time{}
-}
-
-// Only Unavailable proves a transport failure — a request deadline or an
-// application-level error can fire on a healthy connection.
-func isTransportError(err error) bool {
-	return status.Code(err) == codes.Unavailable
-}
-
-// resolve runs without d.mu held so a slow master cannot stall cached-client callers.
-func (d *vectorDiscovery) resolve() {
-	workers := d.discover(d.masters, d.grpcDialOption)
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	// Measured from lookup end so a slow lookup cannot leave the next one already due.
-	defer func() {
-		interval := vectorRecheckInterval
-		if d.client != nil {
-			interval = vectorResolveInterval
-		}
-		d.nextResolve = time.Now().Add(interval)
-		close(d.resolving)
-		d.resolving = nil
-	}()
-	if len(workers) == 0 {
-		return
-	}
-	worker := workers[0]
-	for _, w := range workers {
-		if w == d.worker {
-			worker = w
-			break
-		}
-	}
-	if worker != d.worker {
-		conn, err := pb.GrpcDial(context.Background(), worker, false, d.grpcDialOption)
-		if err != nil {
-			glog.Warningf("S3 Vectors: dial weed-vector %s: %v", worker, err)
-			return
-		}
-		if d.conn != nil {
-			d.conn.Close()
-		}
-		d.worker, d.conn = worker, conn
-		d.client = vector_pb.NewVectorServiceClient(conn)
-	}
-}
-
-// A live client refreshes membership at most every vectorResolveInterval.
-// With no usable backend the recheck is much more frequent: a worker that
-// registers a moment after a lookup must surface inside a caller's retry
-// budget, while an absent one still cannot flood the master with lookups.
-const (
-	vectorResolveInterval = 5 * time.Second
-	vectorRecheckInterval = 500 * time.Millisecond
-)
-
-var errNoVectorWorker = status.Error(codes.Unavailable, "no weed-vector registered on master")
-
-func (d *vectorDiscovery) cli() (vector_pb.VectorServiceClient, error) {
-	d.mu.Lock()
-	if d.resolving == nil && !time.Now().Before(d.nextResolve) {
-		d.resolving = make(chan struct{})
-		go d.resolve()
-	}
-	cli, resolving := d.client, d.resolving
-	d.mu.Unlock()
-	if cli != nil {
-		return cli, nil
-	}
-	// No cached client: wait for the in-flight lookup instead of failing.
-	if resolving != nil {
-		<-resolving
-		d.mu.Lock()
-		cli = d.client
-		d.mu.Unlock()
-		if cli != nil {
-			return cli, nil
-		}
-	}
-	return nil, errNoVectorWorker
-}
-
-func (d *vectorDiscovery) CreateIndex(ctx context.Context, in *vector_pb.CreateIndexRequest, opts ...grpc.CallOption) (resp *vector_pb.CreateIndexResponse, err error) {
-	for attempt := 0; attempt < 2; attempt++ {
-		var c vector_pb.VectorServiceClient
-		if c, err = d.cli(); err != nil {
-			return nil, err
-		}
-		if resp, err = c.CreateIndex(ctx, in, opts...); err == nil {
-			return resp, nil
-		}
-		if !isTransportError(err) {
-			return nil, err
-		}
-		d.reset(c)
-	}
-	return nil, err
-}
-func (d *vectorDiscovery) DropIndex(ctx context.Context, in *vector_pb.DropIndexRequest, opts ...grpc.CallOption) (resp *vector_pb.DropIndexResponse, err error) {
-	for attempt := 0; attempt < 2; attempt++ {
-		var c vector_pb.VectorServiceClient
-		if c, err = d.cli(); err != nil {
-			return nil, err
-		}
-		if resp, err = c.DropIndex(ctx, in, opts...); err == nil {
-			return resp, nil
-		}
-		if !isTransportError(err) {
-			return nil, err
-		}
-		d.reset(c)
-	}
-	return nil, err
-}
-func (d *vectorDiscovery) PutVectors(ctx context.Context, in *vector_pb.PutVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.PutVectorsResponse, err error) {
-	for attempt := 0; attempt < 2; attempt++ {
-		var c vector_pb.VectorServiceClient
-		if c, err = d.cli(); err != nil {
-			return nil, err
-		}
-		if resp, err = c.PutVectors(ctx, in, opts...); err == nil {
-			return resp, nil
-		}
-		if !isTransportError(err) {
-			return nil, err
-		}
-		d.reset(c)
-	}
-	return nil, err
-}
-func (d *vectorDiscovery) GetVectors(ctx context.Context, in *vector_pb.GetVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.GetVectorsResponse, err error) {
-	for attempt := 0; attempt < 2; attempt++ {
-		var c vector_pb.VectorServiceClient
-		if c, err = d.cli(); err != nil {
-			return nil, err
-		}
-		if resp, err = c.GetVectors(ctx, in, opts...); err == nil {
-			return resp, nil
-		}
-		if !isTransportError(err) {
-			return nil, err
-		}
-		d.reset(c)
-	}
-	return nil, err
-}
-func (d *vectorDiscovery) DeleteVectors(ctx context.Context, in *vector_pb.DeleteVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.DeleteVectorsResponse, err error) {
-	for attempt := 0; attempt < 2; attempt++ {
-		var c vector_pb.VectorServiceClient
-		if c, err = d.cli(); err != nil {
-			return nil, err
-		}
-		if resp, err = c.DeleteVectors(ctx, in, opts...); err == nil {
-			return resp, nil
-		}
-		if !isTransportError(err) {
-			return nil, err
-		}
-		d.reset(c)
-	}
-	return nil, err
-}
-func (d *vectorDiscovery) ListVectors(ctx context.Context, in *vector_pb.ListVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.ListVectorsResponse, err error) {
-	for attempt := 0; attempt < 2; attempt++ {
-		var c vector_pb.VectorServiceClient
-		if c, err = d.cli(); err != nil {
-			return nil, err
-		}
-		if resp, err = c.ListVectors(ctx, in, opts...); err == nil {
-			return resp, nil
-		}
-		if !isTransportError(err) {
-			return nil, err
-		}
-		d.reset(c)
-	}
-	return nil, err
-}
-func (d *vectorDiscovery) QueryVectors(ctx context.Context, in *vector_pb.QueryVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.QueryVectorsResponse, err error) {
-	for attempt := 0; attempt < 2; attempt++ {
-		var c vector_pb.VectorServiceClient
-		if c, err = d.cli(); err != nil {
-			return nil, err
-		}
-		if resp, err = c.QueryVectors(ctx, in, opts...); err == nil {
-			return resp, nil
-		}
-		if !isTransportError(err) {
-			return nil, err
-		}
-		d.reset(c)
-	}
-	return nil, err
-}
-
 // isVectorsRequest routes by the SigV4 credential scope — the s3vectors
 // service name lives there, not in the path.
 func isVectorsRequest(r *http.Request) bool {
@@ -1134,32 +905,3 @@ func (v lanceCredentialVendor) VendTableCredentials(ctx context.Context, princip
 // discoverVectorWorkers returns every registered weed-vector address —
 // callers keep their current worker when it is still listed so
 // membership-order changes cannot churn live connections.
-func discoverVectorWorkers(masters []pb.ServerAddress, grpcDialOption grpc.DialOption) []string {
-	for _, master := range masters {
-		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-		conn, err := pb.GrpcDial(ctx, master.ToGrpcAddress(), false, grpcDialOption)
-		if err != nil {
-			cancel()
-			continue
-		}
-		resp, err := master_pb.NewSeaweedClient(conn).ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
-			ClientType: "vector",
-		})
-		conn.Close()
-		cancel()
-		if err != nil {
-			continue
-		}
-		workers := make([]string, 0, len(resp.ClusterNodes))
-		for _, node := range resp.ClusterNodes {
-			glog.V(0).Infof("S3 Vectors: discovered weed-vector at %s via master %s", node.Address, master)
-			workers = append(workers, node.Address)
-		}
-		// A follower can answer with an empty list — workers register with
-		// the leader, so keep looking rather than accept "none."
-		if len(workers) > 0 {
-			return workers
-		}
-	}
-	return nil
-}

weed/command/shell_s3vectors.go

@@ -0,0 +1,101 @@
+package command
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"slices"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/lance"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
+	"github.com/seaweedfs/seaweedfs/weed/shell"
+)
+
+func init() { shell.Commands = append(shell.Commands, &commandS3Vectors{}) }
+
+type commandS3Vectors struct{}
+
+func (c *commandS3Vectors) Name() string                 { return "s3vectors" }
+func (c *commandS3Vectors) HasTag(shell.CommandTag) bool { return false }
+func (c *commandS3Vectors) Help() string {
+	return `manage vector buckets, indexes, policies, tags and vectors
+
+s3vectors -operation CreateVectorBucket -request '{"vectorBucketName":"embeddings"}'
+s3vectors -operation ListVectorBuckets
+s3vectors -operation CreateIndex -file index.json
+s3vectors -operation QueryVectors -file query.json
+
+Requests use the S3 Vectors API JSON shape. The shell uses trusted cluster
+credentials. Index and vector operations require a registered vector engine.
+`
+}
+
+func (c *commandS3Vectors) Do(args []string, env *shell.CommandEnv, writer io.Writer) error {
+	flags := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	operation := flags.String("operation", "", "S3 Vectors operation")
+	request := flags.String("request", "", "request JSON")
+	file := flags.String("file", "", "request JSON file")
+	if err := flags.Parse(args); err != nil {
+		return err
+	}
+	if !slices.Contains(s3vectors.Operations(), *operation) {
+		return fmt.Errorf("unknown operation %q; supported: %v", *operation, s3vectors.Operations())
+	}
+	if *request != "" && *file != "" {
+		return fmt.Errorf("use either -request or -file")
+	}
+	body := []byte(*request)
+	if *file != "" {
+		reader, err := os.Open(*file)
+		if err != nil {
+			return err
+		}
+		defer reader.Close()
+		body, err = io.ReadAll(io.LimitReader(reader, s3_constants.S3VectorsMaxRequestBodyBytes+1))
+		if err != nil {
+			return err
+		}
+	}
+	if len(body) == 0 {
+		body = []byte(`{}`)
+	}
+	if len(body) > s3_constants.S3VectorsMaxRequestBodyBytes {
+		return fmt.Errorf("request is too large")
+	}
+	if !json.Valid(body) {
+		return fmt.Errorf("request must be valid JSON")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	backend := s3vectors.NewVectorDiscovery(env.MasterClient.ListCachedMasters, env.GrpcDialOption(), env.MasterClient.FilerGroup)
+	defer backend.Close()
+	server := s3vectors.NewServer(env, nil, backend)
+	server.SetRegion(s3tables.DefaultRegion)
+	server.SetAccountID(s3tables.DefaultAccountID)
+	catalog := lance.NewServer(env, nil)
+	server.SetDescribe(func(ctx context.Context, id []string) (string, map[string]string, error) {
+		r := (&http.Request{Method: http.MethodPost, Header: make(http.Header)}).WithContext(ctx)
+		return catalog.DescribeTableInternal(r, id)
+	})
+	var response json.RawMessage
+	if err := server.Execute(ctx, *operation, json.RawMessage(body), &response, s3_constants.AccountAdminId); err != nil {
+		return err
+	}
+	if len(response) > 0 {
+		var value interface{}
+		if err := json.Unmarshal(response, &value); err != nil {
+			return err
+		}
+		encoder := json.NewEncoder(writer)
+		encoder.SetIndent("", "  ")
+		return encoder.Encode(value)
+	}
+	return nil
+}

weed/s3api/auth_credentials.go

@@ -2748,11 +2748,13 @@ func (iam *IdentityAccessManagement) evaluateAttachedIAMPolicies(r *http.Request
 	if isBucketListingRequest(r, action) {
 		resourceObject = ""
 	}
-	resource := buildResourceARN(bucket, resourceObject)
+	s3Action, resource := resolveS3AuthTarget(action, bucket, resourceObject, r)
 	principal := buildPrincipalARN(identity, r)
-	s3Action := ResolveS3Action(r, string(action), bucket, resourceObject)
 	explicitAllow := false
 	conditions := engine.ExtractConditionValuesFromRequest(r)
+	for key, values := range vectorIAMConditions(r) {
+		conditions[key] = values
+	}
 	for k, v := range policy_engine.ExtractPrincipalVariables(principal) {
 		conditions[k] = v
 	}
@@ -3214,7 +3216,7 @@ func (iam *IdentityAccessManagement) authorizeWithIAM(r *http.Request, identity
 	// The evaluation below consults the bucket-policy:<bucket> mirror, so the
 	// bucket's lazy load (which backfills that mirror) must happen first -
 	// nothing earlier on a denied request's path would ever trigger it.
-	if iam.primeBucketForIAM != nil && bucket != "" {
+	if iam.primeBucketForIAM != nil && bucket != "" && !strings.HasPrefix(string(action), "s3vectors:") {
 		iam.primeBucketForIAM(bucket)
 	}
 
@@ -3276,6 +3278,9 @@ func (iam *IdentityAccessManagement) authorizeWithIAM(r *http.Request, identity
 // AuthorizeAction applies, so the native-permission floor's explicit-deny
 // check evaluates the same action and resource ARN as the policy engine.
 func resolveS3AuthTarget(action Action, bucket, object string, r *http.Request) (s3Action, resourceArn string) {
+	if strings.HasPrefix(string(action), "s3vectors:") {
+		return string(action), bucket
+	}
 	resourceObjectKey := object
 	if isBucketListingRequest(r, action) {
 		resourceObjectKey = ""

weed/s3api/s3_iam_middleware.go

@@ -257,7 +257,6 @@ func (s3iam *S3IAMIntegration) AuthorizeAction(ctx context.Context, identity *IA
 	}
 
 	// Build resource ARN for the S3 operation
-	resourceArn := buildS3ResourceArn(bucket, resourceObjectKey)
 
 	// Add identity claims to request context for policy variables
 	// Only add claim keys if they don't already exist (to avoid overwriting request-derived context)
@@ -283,7 +282,7 @@ func (s3iam *S3IAMIntegration) AuthorizeAction(ctx context.Context, identity *IA
 	// Determine the specific S3 action based on the HTTP request details. The
 	// prefix promoted into objectKey is not part of the URL; resolve against
 	// the bucket-level object so ?versions and ?uploads keep their own action.
-	specificAction := ResolveS3Action(r, string(action), bucket, resourceObjectKey)
+	specificAction, resourceArn := resolveS3AuthTarget(action, bucket, resourceObjectKey, r)
 
 	// Create action request
 	actionRequest := &integration.ActionRequest{
@@ -397,6 +396,9 @@ func (s3iam *S3IAMIntegration) extractRequestContext(r *http.Request) map[string
 		context["referer"] = referer
 	}
 
+	for key, values := range vectorIAMConditions(r) {
+		context[key] = values
+	}
 	return context
 }
 

weed/s3api/s3api_embedded_iam.go

@@ -1125,8 +1125,24 @@ func iamPolicyArn(policyName string) string {
 // res[5] contains the bucket and optional path after :::
 func (e *EmbeddedIamApi) getActions(policy *policy_engine.PolicyDocument) ([]string, error) {
 	var actions []string
+	hasVectorPolicy := false
 
 	for _, statement := range policy.Statement {
+		vectorOnly := len(statement.Action.Strings()) > 0
+		for _, action := range statement.Action.Strings() {
+			if !strings.HasPrefix(action, "s3vectors:") {
+				vectorOnly = false
+			}
+		}
+		if vectorOnly && (statement.Effect == policy_engine.PolicyEffectAllow || statement.Effect == policy_engine.PolicyEffectDeny) {
+			for _, resource := range statement.Resource.Strings() {
+				if resource == "*" || strings.HasPrefix(resource, "arn:aws:s3vectors:") {
+					hasVectorPolicy = true
+				}
+			}
+			continue
+		}
+
 		if statement.Effect != policy_engine.PolicyEffectAllow {
 			return nil, fmt.Errorf("not a valid effect: '%s'. Only 'Allow' is possible", statement.Effect)
 		}
@@ -1181,7 +1197,7 @@ func (e *EmbeddedIamApi) getActions(policy *policy_engine.PolicyDocument) ([]str
 		}
 	}
 
-	if len(actions) == 0 {
+	if len(actions) == 0 && !hasVectorPolicy {
 		return nil, fmt.Errorf("no valid actions found in policy document")
 	}
 	return actions, nil

weed/s3api/s3api_status_handlers.go

@@ -1,12 +1,18 @@
 package s3api
 
 import (
+	"encoding/json"
 	"net/http"
 
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 )
 
 func (s3a *S3ApiServer) StatusHandler(w http.ResponseWriter, r *http.Request) {
+	if _, features := r.URL.Query()["features"]; features {
+		body, _ := json.Marshal(map[string]bool{"sseS3": GetSSES3KeyManager().IsConfigured()})
+		s3err.WriteResponse(w, r, http.StatusOK, body, "application/json")
+		return
+	}
 	// write out the response code and content type header
 	s3err.WriteResponse(w, r, http.StatusOK, []byte{}, "")
 }

weed/s3api/s3api_vector_iam.go

@@ -0,0 +1,49 @@
+package s3api
+
+import (
+	"context"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+	"net/http"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+)
+
+func (s3a *S3ApiServer) EvaluateIAMPolicy(r *http.Request, action, resource string, policyContext *s3tables.PolicyContext) (allowed, denied bool) {
+	identity, _ := s3_constants.GetIdentityFromContext(r).(*Identity)
+	if identity == nil || s3a.iam == nil {
+		return false, false
+	}
+	conditions := make(map[string][]string)
+	for key, value := range policyContext.ResourceTags {
+		conditions["aws:ResourceTag/"+key] = []string{value}
+	}
+	for key, value := range policyContext.TableBucketTags {
+		conditions["s3vectors:VectorBucketTag/"+key] = []string{value}
+	}
+	for key, value := range policyContext.RequestTags {
+		conditions["aws:RequestTag/"+key] = []string{value}
+	}
+	if len(policyContext.TagKeys) > 0 {
+		conditions["aws:TagKeys"] = policyContext.TagKeys
+	}
+	r = r.WithContext(context.WithValue(r.Context(), vectorIAMConditionsKey{}, conditions))
+
+	result := s3a.iam.evaluateAttachedIAMPolicies(r, identity, Action(action), resource, "")
+	denied = result == attachedIAMPolicyDeny || s3a.iam.isActionExplicitlyDeniedByIAM(r, identity, buildPrincipalARN(identity, r), action, resource)
+	if denied {
+		return false, true
+	}
+	if s3a.iam.authorizationRoute(r, identity) == authorizeViaIAMIntegration {
+		allowed = s3a.iam.VerifyActionPermission(r, identity, Action(action), resource, "") == s3err.ErrNone
+		return allowed, hasSessionToken(r) && !allowed
+	}
+	return result == attachedIAMPolicyAllow, false
+}
+
+type vectorIAMConditionsKey struct{}
+
+func vectorIAMConditions(r *http.Request) map[string][]string {
+	conditions, _ := r.Context().Value(vectorIAMConditionsKey{}).(map[string][]string)
+	return conditions
+}

weed/s3api/s3api_vector_iam_test.go

@@ -0,0 +1,62 @@
+package s3api
+
+import (
+	"fmt"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVectorIAMPolicies(t *testing.T) {
+	const resource = "arn:aws:s3vectors:us-east-1:000000000000:bucket/embeddings/index/docs"
+	for _, action := range []string{"s3vectors:QueryVectors", "s3vectors:*"} {
+		t.Run(action, func(t *testing.T) {
+			iam := &IdentityAccessManagement{isAuthEnabled: true}
+			require.NoError(t, iam.PutPolicy("vectors", fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":%q,"Resource":%q,"Condition":{"StringEquals":{"aws:ResourceTag/team":"search"}}}]}`, action, resource)))
+			identity := &Identity{Name: "reader", PolicyNames: []string{"vectors"}}
+			r := httptest.NewRequest("POST", "/QueryVectors", nil)
+			r = r.WithContext(s3_constants.SetIdentityInContext(r.Context(), identity))
+			server := &S3ApiServer{iam: iam}
+			allowed, denied := server.EvaluateIAMPolicy(r, "s3vectors:QueryVectors", resource, &s3tables.PolicyContext{ResourceTags: map[string]string{"team": "search"}})
+			require.True(t, allowed)
+			require.False(t, denied)
+			allowed, denied = server.EvaluateIAMPolicy(r, "s3vectors:QueryVectors", resource, &s3tables.PolicyContext{ResourceTags: map[string]string{"team": "other"}})
+			require.False(t, allowed)
+			require.False(t, denied)
+			allowed, _ = server.EvaluateIAMPolicy(r, "s3vectors:QueryVectors", resource+"-other", &s3tables.PolicyContext{ResourceTags: map[string]string{"team": "search"}})
+			require.False(t, allowed)
+			require.NoError(t, iam.PutPolicy("deny", `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"s3vectors:QueryVectors","Resource":"*"}]}`))
+			identity.PolicyNames = append(identity.PolicyNames, "deny")
+			allowed, denied = server.EvaluateIAMPolicy(r, "s3vectors:QueryVectors", resource, &s3tables.PolicyContext{ResourceTags: map[string]string{"team": "search"}})
+			require.False(t, allowed)
+			require.True(t, denied)
+		})
+	}
+}
+
+func TestEmbeddedIAMAcceptsVectorPoliciesWithoutFlattening(t *testing.T) {
+	api := &EmbeddedIamApi{}
+	for _, effect := range []string{"Allow", "Deny"} {
+		raw := fmt.Sprintf(`{"Statement":[{"Effect":%q,"Action":"s3vectors:*","Resource":"arn:aws:s3vectors:us-east-1:000000000000:bucket/embeddings/*"}]}`, effect)
+		policy, err := api.GetPolicyDocument(&raw)
+		require.NoError(t, err)
+		actions, err := api.getActions(&policy)
+		require.NoError(t, err)
+		require.Empty(t, actions)
+	}
+}
+
+func TestVectorIAMBucketTagDeny(t *testing.T) {
+	iam := &IdentityAccessManagement{isAuthEnabled: true}
+	require.NoError(t, iam.PutPolicy("vectors", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3vectors:*","Resource":"*"},{"Effect":"Deny","Action":"s3vectors:QueryVectors","Resource":"*","Condition":{"StringEquals":{"s3vectors:VectorBucketTag/classification":"private"}}}]}`))
+	identity := &Identity{Name: "reader", PolicyNames: []string{"vectors"}}
+	r := httptest.NewRequest("POST", "/QueryVectors", nil)
+	r = r.WithContext(s3_constants.SetIdentityInContext(r.Context(), identity))
+	server := &S3ApiServer{iam: iam}
+	allowed, denied := server.EvaluateIAMPolicy(r, "s3vectors:QueryVectors", "arn:aws:s3vectors:us-east-1:000000000000:bucket/embeddings/index/docs", &s3tables.PolicyContext{TableBucketTags: map[string]string{"classification": "private"}})
+	require.False(t, allowed)
+	require.True(t, denied)
+}

weed/s3api/s3vectors/create_recovery_test.go

@@ -0,0 +1,293 @@
+package s3vectors
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/vector_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type failingCreateVectors struct {
+	*fakeVectors
+	err      error
+	onCreate func()
+}
+
+func (f *failingCreateVectors) CreateIndex(_ context.Context, request *vector_pb.CreateIndexRequest, _ ...grpc.CallOption) (*vector_pb.CreateIndexResponse, error) {
+	f.created = append(f.created, request)
+	err := f.err
+	if f.onCreate != nil {
+		f.onCreate()
+	}
+	return nil, err
+}
+
+func TestCreateIndexPreservesAnUnknownOutcome(t *testing.T) {
+	for _, code := range []codes.Code{codes.Unavailable, codes.DeadlineExceeded, codes.Canceled, codes.Internal, codes.InvalidArgument} {
+		t.Run(code.String(), func(t *testing.T) {
+			fs := s3tablestest.Start(t)
+			engine := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(code, "response lost")}
+			server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
+			postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+			result := postJSON(t, server.handleCreateIndex, &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "cosine"})
+			if result.Code == http.StatusOK {
+				t.Fatal("failed engine reported success")
+			}
+			got := postJSON(t, server.handleGetIndex, &GetIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx"})
+			want := http.StatusOK
+			if code == codes.InvalidArgument {
+				want = http.StatusNotFound
+			}
+			if got.Code != want {
+				t.Fatalf("catalog after %s: %d %s", code, got.Code, got.Body)
+			}
+		})
+	}
+}
+
+func TestCreateIndexRetainsTheWriteErrorIfRediscoveryFails(t *testing.T) {
+	engine := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(codes.Unavailable, "response lost after commit")}
+	discovery := &vectorDiscovery{client: engine, nextResolve: time.Now().Add(time.Hour), discover: func([]pb.ServerAddress, grpc.DialOption) []string { return nil }}
+	_, err := discovery.CreateIndex(t.Context(), &vector_pb.CreateIndexRequest{})
+	if status.Convert(err).Message() != "response lost after commit" || len(engine.created) != 1 {
+		t.Fatalf("ambiguous result replaced by lookup failure: %v", err)
+	}
+}
+
+func TestCreateIndexResumesAnUnknownOutcome(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	engine := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(codes.Unavailable, "request lost")}
+	server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
+	postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+	request := &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "cosine",
+		MetadataConfiguration: &MetadataConfiguration{NonFilterableMetadataKeys: []string{"b", "a"}}, Tags: map[string]string{"team": "search"}}
+	if result := postJSON(t, server.handleCreateIndex, request); result.Code == http.StatusOK {
+		t.Fatal("unavailable engine reported success")
+	}
+	before, err := server.indexRecord(t.Context(), "vec-bucket", "idx")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, change := range []func(*CreateIndexRequest){
+		func(r *CreateIndexRequest) { r.Dimension++ },
+		func(r *CreateIndexRequest) { r.DistanceMetric = "euclidean" },
+		func(r *CreateIndexRequest) { r.MetadataConfiguration = nil },
+	} {
+		other := *request
+		change(&other)
+		if result := postJSON(t, server.handleCreateIndex, &other); result.Code != http.StatusConflict {
+			t.Fatalf("incompatible retry: %d %s", result.Code, result.Body)
+		}
+	}
+	if len(engine.created) != 1 {
+		t.Fatal("incompatible retry reached the engine")
+	}
+	engine.err = status.Error(codes.InvalidArgument, "retry refused")
+	postJSON(t, server.handleCreateIndex, request)
+	if _, err := server.indexRecord(t.Context(), "vec-bucket", "idx"); err != nil {
+		t.Fatalf("retry removed an uncertain dataset's catalog: %v", err)
+	}
+	engine.err = nil
+	request.MetadataConfiguration.NonFilterableMetadataKeys = []string{"a", "b"}
+	if result := postJSON(t, server.handleCreateIndex, request); result.Code != http.StatusOK {
+		t.Fatalf("resume: %d %s", result.Code, result.Body)
+	}
+	after, err := server.indexRecord(t.Context(), "vec-bucket", "idx")
+	if err != nil || recordNonce(before) != recordNonce(after) {
+		t.Fatalf("resume replaced the catalog: %v", err)
+	}
+	if result := postJSON(t, server.handleCreateIndex, request); result.Code != http.StatusConflict {
+		t.Fatalf("completed index was initialized again: %d %s", result.Code, result.Body)
+	}
+	result := postJSON(t, server.handleListTagsForResource, &ListTagsForResourceRequest{ResourceArn: server.indexArn("vec-bucket", "idx")})
+	var tags ListTagsForResourceResponse
+	if err := json.Unmarshal(result.Body.Bytes(), &tags); err != nil || tags.Tags["team"] != "search" {
+		t.Fatalf("resume lost tags: %s (%v)", result.Body, err)
+	}
+}
+
+func TestCreateIndexRollbackPreservesACompletedRetry(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	engine := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(codes.InvalidArgument, "first attempt failed")}
+	server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
+	postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+	request := &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "cosine"}
+	engine.onCreate = func() {
+		engine.onCreate = nil
+		engine.err = nil
+		if result := postJSON(t, server.handleCreateIndex, request); result.Code != http.StatusOK {
+			t.Fatalf("concurrent retry: %d %s", result.Code, result.Body)
+		}
+	}
+	if result := postJSON(t, server.handleCreateIndex, request); result.Code == http.StatusOK {
+		t.Fatal("first attempt reported success")
+	}
+	if _, err := server.indexRecord(t.Context(), "vec-bucket", "idx"); err != nil {
+		t.Fatalf("rollback removed the completed retry: %v", err)
+	}
+	if result := postJSON(t, server.handleCreateIndex, request); result.Code != http.StatusConflict {
+		t.Fatalf("completed retry lost its state: %d %s", result.Code, result.Body)
+	}
+}
+
+func TestCreateIndexRetainsAnUnknownOutcomeAfterARefusedRetry(t *testing.T) {
+	for _, code := range []codes.Code{codes.PermissionDenied, codes.FailedPrecondition} {
+		t.Run(code.String(), func(t *testing.T) {
+			fs := s3tablestest.Start(t)
+			first := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(codes.Unavailable, "response lost after commit")}
+			second := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(code, "retry refused")}
+			discovery := &vectorDiscovery{client: first, nextResolve: time.Now().Add(time.Hour)}
+			discovery.discover = func([]pb.ServerAddress, grpc.DialOption) []string {
+				discovery.mu.Lock()
+				discovery.client = second
+				discovery.mu.Unlock()
+				return nil
+			}
+			defer discovery.Close()
+			server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, discovery)
+			postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+			result := postJSON(t, server.handleCreateIndex, &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "cosine"})
+			if result.Code != http.StatusServiceUnavailable || len(first.created) != 1 || len(second.created) != 1 {
+				t.Fatalf("unknown outcome lost: %d %s (attempts %d, %d)", result.Code, result.Body, len(first.created), len(second.created))
+			}
+			if _, err := server.indexRecord(t.Context(), "vec-bucket", "idx"); err != nil {
+				t.Fatalf("refused retry erased a possibly committed dataset: %v", err)
+			}
+		})
+	}
+}
+
+type concurrentCreateVectors struct {
+	vector_pb.VectorServiceClient
+	create func() (*vector_pb.CreateIndexResponse, error)
+}
+
+func (f *concurrentCreateVectors) CreateIndex(context.Context, *vector_pb.CreateIndexRequest, ...grpc.CallOption) (*vector_pb.CreateIndexResponse, error) {
+	return f.create()
+}
+
+func TestCreateIndexRollbackPreservesAnActiveRetry(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	firstStarted, retryStarted, finishRetry := make(chan struct{}), make(chan struct{}), make(chan struct{})
+	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+	defer cancel()
+	var calls atomic.Int32
+	engine := &concurrentCreateVectors{create: func() (*vector_pb.CreateIndexResponse, error) {
+		if calls.Add(1) == 1 {
+			close(firstStarted)
+			select {
+			case <-retryStarted:
+				return nil, status.Error(codes.InvalidArgument, "first attempt refused")
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			}
+		}
+		close(retryStarted)
+		select {
+		case <-finishRetry:
+			return &vector_pb.CreateIndexResponse{}, nil
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}}
+	server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
+	postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+	request := &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "cosine"}
+	first, retry := make(chan *httptest.ResponseRecorder, 1), make(chan *httptest.ResponseRecorder, 1)
+	go func() { first <- postJSON(t, server.handleCreateIndex, request) }()
+	select {
+	case <-firstStarted:
+	case <-ctx.Done():
+		t.Fatal("first create did not reach the engine")
+	}
+	go func() { retry <- postJSON(t, server.handleCreateIndex, request) }()
+	if result := <-first; result.Code == http.StatusOK {
+		t.Fatal("first attempt reported success")
+	}
+	if _, err := server.indexRecord(t.Context(), "vec-bucket", "idx"); err != nil {
+		t.Fatalf("rollback removed an active retry: %v", err)
+	}
+	close(finishRetry)
+	if result := <-retry; result.Code != http.StatusOK {
+		t.Fatalf("retry could not finish: %d %s", result.Code, result.Body)
+	}
+}
+
+func TestCreateIndexResumesWithTheDefaultMetric(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	engine := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(codes.Unavailable, "request lost")}
+	server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
+	postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+	request := &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3}
+	postJSON(t, server.handleCreateIndex, request)
+	engine.err = nil
+	if result := postJSON(t, server.handleCreateIndex, request); result.Code != http.StatusOK {
+		t.Fatalf("default-metric recovery: %d %s", result.Code, result.Body)
+	}
+	if len(engine.created) != 2 || engine.created[1].DistanceMetric != vector_pb.DistanceMetric_L2 {
+		t.Fatal("recovery changed the default metric")
+	}
+}
+
+func TestCreateIndexRetryPreservesDuplicateMetadataKeys(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	engine := &failingCreateVectors{fakeVectors: &fakeVectors{}, err: status.Error(codes.Unavailable, "response lost")}
+	server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
+	postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+	request := &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "cosine",
+		MetadataConfiguration: &MetadataConfiguration{NonFilterableMetadataKeys: []string{"b", "a", "a"}}}
+	postJSON(t, server.handleCreateIndex, request)
+	engine.err = nil
+	if result := postJSON(t, server.handleCreateIndex, request); result.Code != http.StatusOK {
+		t.Fatalf("duplicate-key recovery: %d %s", result.Code, result.Body)
+	}
+	if len(engine.created) != 2 {
+		t.Fatalf("engine calls: %d", len(engine.created))
+	}
+	for _, call := range engine.created {
+		keys := slices.Clone(call.NonFilterableMetadataKeys)
+		slices.Sort(keys)
+		if !slices.Equal(keys, []string{"a", "a", "b"}) {
+			t.Fatalf("initialization changed metadata keys: %q", keys)
+		}
+	}
+}
+
+func TestCreateIndexDoesNotInitializeAConcurrentCatalogCreate(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	engine := &fakeVectors{}
+	server := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine)
+	postJSON(t, server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"})
+	postJSON(t, server.handleCreateIndex, &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "seed", DataType: "float32", Dimension: 7, DistanceMetric: "cosine"})
+	dir, name := splitEntryPath(s3tables.GetTablePath("vec-bucket", indexNamespace, "idx"))
+	seed := fs.Get(dir, "seed")
+	var lookups int
+	fs.OnLookup = func(d, n string) {
+		if d == dir && n == name {
+			lookups++
+			if lookups == 2 {
+				fs.Put(dir, name, seed.Extended)
+			}
+		}
+	}
+	result := postJSON(t, server.handleCreateIndex, &CreateIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "cosine"})
+	if result.Code != http.StatusConflict || len(engine.created) != 1 {
+		t.Fatalf("initialized a competing record: %d %s (%d engine calls)", result.Code, result.Body, len(engine.created))
+	}
+	record, err := server.indexRecord(t.Context(), "vec-bucket", "idx")
+	if err != nil || indexFromRecord(server, "vec-bucket", "idx", record).Dimension != 7 {
+		t.Fatalf("changed the competing record: %v", err)
+	}
+}

weed/s3api/s3vectors/discovery.go

@@ -0,0 +1,358 @@
+package s3vectors
+
+import (
+	"context"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/vector_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"sync"
+	"time"
+)
+
+func NewVectorDiscovery(masters func() []pb.ServerAddress, dialOption grpc.DialOption, filerGroup string) *vectorDiscovery {
+	return &vectorDiscovery{masters: masters, grpcDialOption: dialOption, discover: func(m []pb.ServerAddress, opt grpc.DialOption) []string {
+		return discoverVectorWorkers(m, opt, filerGroup)
+	}}
+}
+
+type vectorDiscovery struct {
+	masters        func() []pb.ServerAddress
+	grpcDialOption grpc.DialOption
+	discover       func([]pb.ServerAddress, grpc.DialOption) []string
+	mu             sync.Mutex
+	closed         bool
+	worker         string
+	failed         string
+	active         map[vector_pb.VectorServiceClient]int
+	retired        map[vector_pb.VectorServiceClient]*grpc.ClientConn
+	conn           *grpc.ClientConn
+	client         vector_pb.VectorServiceClient
+	nextResolve    time.Time
+	resolving      chan struct{} // closed when the in-flight resolve lands
+}
+
+func (d *vectorDiscovery) Close() error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	d.closed = true
+	d.retire()
+	d.worker, d.conn, d.client = "", nil, nil
+	return nil
+}
+
+func (d *vectorDiscovery) reset(c vector_pb.VectorServiceClient) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if c != d.client {
+		return
+	}
+	d.failed = d.worker
+	d.retire()
+	d.worker, d.conn, d.client = "", nil, nil
+	d.nextResolve = time.Time{}
+}
+
+func (d *vectorDiscovery) retire() {
+	if d.conn == nil {
+		return
+	}
+	if d.active[d.client] == 0 {
+		_ = d.conn.Close()
+		return
+	}
+	if d.retired == nil {
+		d.retired = make(map[vector_pb.VectorServiceClient]*grpc.ClientConn)
+	}
+	d.retired[d.client] = d.conn
+}
+
+func (d *vectorDiscovery) release(c vector_pb.VectorServiceClient) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	d.active[c]--
+	if d.active[c] == 0 {
+		delete(d.active, c)
+		if conn := d.retired[c]; conn != nil {
+			_ = conn.Close()
+			delete(d.retired, c)
+		}
+	}
+}
+
+func isTransportError(err error) bool {
+	return status.Code(err) == codes.Unavailable
+}
+
+func (d *vectorDiscovery) resolve() {
+	var masters []pb.ServerAddress
+	if d.masters != nil {
+		masters = d.masters()
+	}
+	workers := d.discover(masters, d.grpcDialOption)
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	// Measured from lookup end so a slow lookup cannot leave the next one already due.
+	defer func() {
+		interval := vectorRecheckInterval
+		if d.client != nil {
+			interval = vectorResolveInterval
+		}
+		d.nextResolve = time.Now().Add(interval)
+		close(d.resolving)
+		d.resolving = nil
+	}()
+	if d.closed || len(workers) == 0 {
+		return
+	}
+	worker := workers[0]
+	for _, w := range workers {
+		if w != d.failed {
+			worker = w
+			break
+		}
+	}
+	for _, w := range workers {
+		if w == d.worker && w != d.failed {
+			worker = w
+			break
+		}
+	}
+	if worker != d.worker {
+		conn, err := pb.GrpcDial(context.Background(), worker, false, d.grpcDialOption)
+		if err != nil {
+			glog.Warningf("S3 Vectors: dial weed-vector %s: %v", worker, err)
+			return
+		}
+		d.retire()
+		d.worker, d.conn = worker, conn
+		d.client = vector_pb.NewVectorServiceClient(conn)
+	}
+}
+
+const (
+	vectorResolveInterval = 5 * time.Second
+	vectorRecheckInterval = 500 * time.Millisecond
+)
+
+var errNoVectorWorker = status.Error(codes.Unavailable, "no weed-vector registered on master")
+
+func (d *vectorDiscovery) cli(ctx context.Context) (vector_pb.VectorServiceClient, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, status.FromContextError(err).Err()
+	}
+	d.mu.Lock()
+	if d.closed {
+		d.mu.Unlock()
+		return nil, status.Error(codes.Canceled, "vector discovery closed")
+	}
+	if d.resolving == nil && !time.Now().Before(d.nextResolve) {
+		d.resolving = make(chan struct{})
+		go d.resolve()
+	}
+	if d.client == nil && d.resolving != nil {
+		resolving := d.resolving
+		d.mu.Unlock()
+		select {
+		case <-resolving:
+		case <-ctx.Done():
+			return nil, status.FromContextError(ctx.Err()).Err()
+		}
+		d.mu.Lock()
+	}
+	defer d.mu.Unlock()
+	if d.closed {
+		return nil, status.Error(codes.Canceled, "vector discovery closed")
+	}
+	if d.client == nil {
+		return nil, errNoVectorWorker
+	}
+	if d.active == nil {
+		d.active = make(map[vector_pb.VectorServiceClient]int)
+	}
+	d.active[d.client]++
+	return d.client, nil
+}
+
+func (d *vectorDiscovery) CreateIndex(ctx context.Context, in *vector_pb.CreateIndexRequest, opts ...grpc.CallOption) (resp *vector_pb.CreateIndexResponse, err error) {
+	var firstErr error
+	for attempt := 0; attempt < 2; attempt++ {
+		var c vector_pb.VectorServiceClient
+		var resolveErr error
+		if c, resolveErr = d.cli(ctx); resolveErr != nil {
+			if attempt > 0 {
+				return nil, firstErr
+			}
+			return nil, resolveErr
+		}
+		resp, err = c.CreateIndex(ctx, in, opts...)
+		if isTransportError(err) {
+			d.reset(c)
+		}
+		d.release(c)
+		if err == nil {
+			return resp, nil
+		}
+		if firstErr == nil {
+			firstErr = err
+		}
+		if !isTransportError(err) {
+			return nil, firstErr
+		}
+	}
+	return nil, firstErr
+}
+func (d *vectorDiscovery) DropIndex(ctx context.Context, in *vector_pb.DropIndexRequest, opts ...grpc.CallOption) (resp *vector_pb.DropIndexResponse, err error) {
+	for attempt := 0; attempt < 2; attempt++ {
+		var c vector_pb.VectorServiceClient
+		if c, err = d.cli(ctx); err != nil {
+			return nil, err
+		}
+		resp, err = c.DropIndex(ctx, in, opts...)
+		if isTransportError(err) {
+			d.reset(c)
+		}
+		d.release(c)
+		if err == nil {
+			return resp, nil
+		}
+		if !isTransportError(err) {
+			return nil, err
+		}
+	}
+	return nil, err
+}
+func (d *vectorDiscovery) PutVectors(ctx context.Context, in *vector_pb.PutVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.PutVectorsResponse, err error) {
+	for attempt := 0; attempt < 2; attempt++ {
+		var c vector_pb.VectorServiceClient
+		if c, err = d.cli(ctx); err != nil {
+			return nil, err
+		}
+		resp, err = c.PutVectors(ctx, in, opts...)
+		if isTransportError(err) {
+			d.reset(c)
+		}
+		d.release(c)
+		if err == nil {
+			return resp, nil
+		}
+		if !isTransportError(err) {
+			return nil, err
+		}
+	}
+	return nil, err
+}
+func (d *vectorDiscovery) GetVectors(ctx context.Context, in *vector_pb.GetVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.GetVectorsResponse, err error) {
+	for attempt := 0; attempt < 2; attempt++ {
+		var c vector_pb.VectorServiceClient
+		if c, err = d.cli(ctx); err != nil {
+			return nil, err
+		}
+		resp, err = c.GetVectors(ctx, in, opts...)
+		if isTransportError(err) {
+			d.reset(c)
+		}
+		d.release(c)
+		if err == nil {
+			return resp, nil
+		}
+		if !isTransportError(err) {
+			return nil, err
+		}
+	}
+	return nil, err
+}
+func (d *vectorDiscovery) DeleteVectors(ctx context.Context, in *vector_pb.DeleteVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.DeleteVectorsResponse, err error) {
+	for attempt := 0; attempt < 2; attempt++ {
+		var c vector_pb.VectorServiceClient
+		if c, err = d.cli(ctx); err != nil {
+			return nil, err
+		}
+		resp, err = c.DeleteVectors(ctx, in, opts...)
+		if isTransportError(err) {
+			d.reset(c)
+		}
+		d.release(c)
+		if err == nil {
+			return resp, nil
+		}
+		if !isTransportError(err) {
+			return nil, err
+		}
+	}
+	return nil, err
+}
+func (d *vectorDiscovery) ListVectors(ctx context.Context, in *vector_pb.ListVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.ListVectorsResponse, err error) {
+	for attempt := 0; attempt < 2; attempt++ {
+		var c vector_pb.VectorServiceClient
+		if c, err = d.cli(ctx); err != nil {
+			return nil, err
+		}
+		resp, err = c.ListVectors(ctx, in, opts...)
+		if isTransportError(err) {
+			d.reset(c)
+		}
+		d.release(c)
+		if err == nil {
+			return resp, nil
+		}
+		if !isTransportError(err) {
+			return nil, err
+		}
+	}
+	return nil, err
+}
+func (d *vectorDiscovery) QueryVectors(ctx context.Context, in *vector_pb.QueryVectorsRequest, opts ...grpc.CallOption) (resp *vector_pb.QueryVectorsResponse, err error) {
+	for attempt := 0; attempt < 2; attempt++ {
+		var c vector_pb.VectorServiceClient
+		if c, err = d.cli(ctx); err != nil {
+			return nil, err
+		}
+		resp, err = c.QueryVectors(ctx, in, opts...)
+		if isTransportError(err) {
+			d.reset(c)
+		}
+		d.release(c)
+		if err == nil {
+			return resp, nil
+		}
+		if !isTransportError(err) {
+			return nil, err
+		}
+	}
+	return nil, err
+}
+
+func discoverVectorWorkers(masters []pb.ServerAddress, grpcDialOption grpc.DialOption, filerGroup string) []string {
+	for _, master := range masters {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		conn, err := pb.GrpcDial(ctx, master.ToGrpcAddress(), false, grpcDialOption)
+		if err != nil {
+			cancel()
+			continue
+		}
+		resp, err := master_pb.NewSeaweedClient(conn).ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
+			ClientType: "vector",
+			FilerGroup: filerGroup,
+		})
+		conn.Close()
+		cancel()
+		if err != nil {
+			continue
+		}
+		workers := make([]string, 0, len(resp.ClusterNodes))
+		for _, node := range resp.ClusterNodes {
+			glog.V(0).Infof("S3 Vectors: discovered weed-vector at %s via master %s", node.Address, master)
+			workers = append(workers, node.Address)
+		}
+		// A follower can answer with an empty list — workers register with
+		// the leader, so keep looking rather than accept "none."
+		if len(workers) > 0 {
+			return workers
+		}
+	}
+	return nil
+}

weed/s3api/s3vectors/discovery_test.go

@@ -1,6 +1,7 @@
-package command
+package s3vectors
 
 import (
+	"context"
 	"fmt"
 	"sync/atomic"
 	"testing"
@@ -26,10 +27,10 @@ func TestVectorDiscoveryRechecksRegistration(t *testing.T) {
 		},
 	}
 
-	if _, err := d.cli(); status.Code(err) != codes.Unavailable {
+	if _, err := d.cli(t.Context()); status.Code(err) != codes.Unavailable {
 		t.Fatalf("no registered worker: expected Unavailable, got %v", err)
 	}
-	if _, err := d.cli(); status.Code(err) != codes.Unavailable {
+	if _, err := d.cli(t.Context()); status.Code(err) != codes.Unavailable {
 		t.Fatalf("no registered worker: expected Unavailable, got %v", err)
 	}
 	if got := atomic.LoadInt32(&calls); got != 1 {
@@ -46,7 +47,7 @@ func TestVectorDiscoveryRechecksRegistration(t *testing.T) {
 	}
 	d.grpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
 
-	if _, err := d.cli(); err != nil {
+	if _, err := d.cli(t.Context()); err != nil {
 		t.Fatalf("worker registered: expected a client, got %v", err)
 	}
 	if got := atomic.LoadInt32(&calls); got != 2 {
@@ -54,7 +55,7 @@ func TestVectorDiscoveryRechecksRegistration(t *testing.T) {
 	}
 
 	// A live client rides out the refresh interval without rediscovery.
-	if _, err := d.cli(); err != nil {
+	if _, err := d.cli(t.Context()); err != nil {
 		t.Fatalf("cached client rejected: %v", err)
 	}
 	if got := atomic.LoadInt32(&calls); got != 2 {
@@ -77,10 +78,10 @@ func TestVectorDiscoveryThrottlesFromLookupEnd(t *testing.T) {
 		},
 	}
 
-	if _, err := d.cli(); status.Code(err) != codes.Unavailable {
+	if _, err := d.cli(t.Context()); status.Code(err) != codes.Unavailable {
 		t.Fatalf("no registered worker: expected Unavailable, got %v", err)
 	}
-	if _, err := d.cli(); status.Code(err) != codes.Unavailable {
+	if _, err := d.cli(t.Context()); status.Code(err) != codes.Unavailable {
 		t.Fatalf("no registered worker: expected Unavailable, got %v", err)
 	}
 	if got := atomic.LoadInt32(&calls); got != 1 {
@@ -95,7 +96,7 @@ func TestVectorDiscoveryThrottlesFromLookupEnd(t *testing.T) {
 		d.mu.Lock()
 		d.nextResolve = time.Time{}
 		d.mu.Unlock()
-		if _, err := d.cli(); err != nil {
+		if _, err := d.cli(t.Context()); err != nil {
 			t.Fatalf("worker registered: expected a client, got %v", err)
 		}
 		waitResolved(d)
@@ -126,7 +127,7 @@ func TestVectorDiscoveryLookupDoesNotBlockCachedClient(t *testing.T) {
 			return []string{"127.0.0.1:9301"}
 		},
 	}
-	cached, err := d.cli()
+	cached, err := d.cli(t.Context())
 	if err != nil {
 		t.Fatalf("worker registered: expected a client, got %v", err)
 	}
@@ -147,7 +148,7 @@ func TestVectorDiscoveryLookupDoesNotBlockCachedClient(t *testing.T) {
 	const callers = 8
 	got := make(chan error, callers+1)
 	call := func() {
-		c, err := d.cli()
+		c, err := d.cli(t.Context())
 		if err == nil && c != cached {
 			err = fmt.Errorf("got a different client")
 		}
@@ -195,7 +196,7 @@ func TestVectorDiscoveryUncachedCallersShareLookup(t *testing.T) {
 	got := make(chan error, callers)
 	for i := 0; i < callers; i++ {
 		go func() {
-			_, err := d.cli()
+			_, err := d.cli(t.Context())
 			got <- err
 		}()
 	}
@@ -219,7 +220,7 @@ func TestVectorDiscoveryKeepsConnOnDialError(t *testing.T) {
 			return []string{"127.0.0.1:9301"}
 		},
 	}
-	cached, err := d.cli()
+	cached, err := d.cli(t.Context())
 	if err != nil {
 		t.Fatalf("worker registered: expected a client, got %v", err)
 	}
@@ -232,7 +233,7 @@ func TestVectorDiscoveryKeepsConnOnDialError(t *testing.T) {
 	d.mu.Lock()
 	d.nextResolve = time.Time{}
 	d.mu.Unlock()
-	if c, err := d.cli(); err != nil || c != cached {
+	if c, err := d.cli(t.Context()); err != nil || c != cached {
 		t.Fatalf("cached client: got %v, %v", c, err)
 	}
 	waitResolved(d)
@@ -243,3 +244,109 @@ func TestVectorDiscoveryKeepsConnOnDialError(t *testing.T) {
 		t.Fatalf("failed dial dropped the cached connection")
 	}
 }
+
+func TestVectorDiscoveryDrainsRetiredConnections(t *testing.T) {
+	d := &vectorDiscovery{grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), discover: func([]pb.ServerAddress, grpc.DialOption) []string { return []string{"127.0.0.1:9301"} }}
+	first, err := d.cli(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := d.cli(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	conn := d.conn
+	d.reset(first)
+	d.release(first)
+	if conn.GetState() == connectivity.Shutdown {
+		t.Fatal("active RPC lost its connection")
+	}
+	d.release(second)
+	if conn.GetState() != connectivity.Shutdown {
+		t.Fatal("retired connection was not closed after the last RPC")
+	}
+}
+
+func TestVectorDiscoveryWaitingRespectsCancellation(t *testing.T) {
+	entered, release := make(chan struct{}), make(chan struct{})
+	d := &vectorDiscovery{discover: func([]pb.ServerAddress, grpc.DialOption) []string { close(entered); <-release; return nil }}
+	ctx, cancel := context.WithCancel(t.Context())
+	result := make(chan error, 1)
+	go func() { _, err := d.cli(ctx); result <- err }()
+	<-entered
+	cancel()
+	select {
+	case err := <-result:
+		if status.Code(err) != codes.Canceled {
+			t.Fatal(err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("request remained blocked after cancellation")
+	}
+	close(release)
+	waitResolved(d)
+}
+
+func TestVectorDiscoveryRetriesAnotherWorker(t *testing.T) {
+	d := &vectorDiscovery{grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), discover: func([]pb.ServerAddress, grpc.DialOption) []string {
+		return []string{"127.0.0.1:9301", "127.0.0.1:9302"}
+	}}
+	first, err := d.cli(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	d.reset(first)
+	d.release(first)
+	second, err := d.cli(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer d.release(second)
+	if d.worker != "127.0.0.1:9302" {
+		t.Fatalf("retry selected failed worker: %s", d.worker)
+	}
+}
+
+func TestVectorDiscoveryCloseDuringLookup(t *testing.T) {
+	entered, release := make(chan struct{}), make(chan struct{})
+	d := &vectorDiscovery{grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), discover: func([]pb.ServerAddress, grpc.DialOption) []string {
+		close(entered)
+		<-release
+		return []string{"127.0.0.1:9301"}
+	}}
+	done := make(chan error, 1)
+	go func() { _, err := d.cli(t.Context()); done <- err }()
+	<-entered
+	_ = d.Close()
+	close(release)
+	if err := <-done; status.Code(err) != codes.Canceled {
+		t.Fatal(err)
+	}
+	if d.conn != nil {
+		t.Fatal("closed discovery opened a connection")
+	}
+}
+
+func TestVectorDiscoveryRefreshesMasterSnapshot(t *testing.T) {
+	masters := []pb.ServerAddress{}
+	d := NewVectorDiscovery(func() []pb.ServerAddress { return masters }, grpc.WithTransportCredentials(insecure.NewCredentials()), "")
+	d.discover = func(current []pb.ServerAddress, _ grpc.DialOption) []string {
+		if len(current) > 0 {
+			return []string{"127.0.0.1:9301"}
+		}
+		return nil
+	}
+	if _, err := d.cli(t.Context()); status.Code(err) != codes.Unavailable {
+		t.Fatal(err)
+	}
+	masters = []pb.ServerAddress{"master:9333"}
+	d.mu.Lock()
+	d.nextResolve = time.Time{}
+	d.mu.Unlock()
+	client, err := d.cli(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	d.release(client)
+	_ = d.Close()
+}

weed/s3api/s3vectors/handlers.go

@@ -295,7 +295,11 @@ func (s *Server) handleCreateVectorBucket(w http.ResponseWriter, r *http.Request
 	// catalog does for CreateTableBucket.
 	principal := s.tables.PrincipalFor(r)
 	allowed := func(action, bucket string) bool {
-		return s3tables.CheckPermissionWithContext(action, principal, s.tables.PrincipalFor(nil), "", "", s.tables.PolicyContext(r, bucket))
+		resource := "*"
+		if bucket != "" {
+			resource = s.vectorBucketArn(bucket)
+		}
+		return s.checkPermission(r, action, principal, s.tables.PrincipalFor(nil), "", resource, s.policyContext(r, bucket, nil, nil, setTags(req.Tags)))
 	}
 	// Tagging names the new bucket, so a grant scoped to it applies.
 	if !allowed("s3vectors:CreateVectorBucket", "") ||
@@ -398,8 +402,8 @@ func (s *Server) handleListVectorBuckets(w http.ResponseWriter, r *http.Request)
 		}
 		arn := s.vectorBucketArn(b.Name)
 		pctx := s.policyContext(r, b.Name, tags, tags, tagRequest{})
-		if !s3tables.CheckPermissionWithContext("s3vectors:ListVectorBuckets", principal, owner, policy, arn, pctx) &&
-			!s3tables.CheckPermissionWithContext("s3vectors:GetVectorBucket", principal, owner, policy, arn, pctx) {
+		if !s.checkPermission(r, "s3vectors:ListVectorBuckets", principal, owner, policy, arn, pctx) &&
+			!s.checkPermission(r, "s3vectors:GetVectorBucket", principal, owner, policy, arn, pctx) {
 			continue
 		}
 		created, _ := time.Parse(time.RFC3339, b.CreatedAt)
@@ -923,7 +927,7 @@ func (s *Server) checkVectorPolicyDoc(r *http.Request, action, bucket string, re
 		return err
 	}
 	principal := s.tables.PrincipalFor(r)
-	if !s3tables.CheckPermissionWithContext(action, principal, record.OwnerAccountID,
+	if !s.checkPermission(r, action, principal, record.OwnerAccountID,
 		vectorPolicyFromExt(ext), s.vectorBucketArn(bucket), s.policyContext(r, bucket, tags, tags, tagRequest{})) {
 		return errVectorDenied
 	}
@@ -956,14 +960,14 @@ func (s *Server) authorizeVectorTags(ctx context.Context, r *http.Request, actio
 	resourceARN, resourceTags := s.vectorBucketArn(bucket), bucketTags
 	if index != "" {
 		resourceARN, resourceTags = s.indexArn(bucket, index), nil
-		if readsResourceTags(policy) {
+		if _, usesIAM := s.authenticator.(IAMAuthorizer); readsResourceTags(policy) || usesIAM {
 			if resourceTags, err = s.indexTags(ctx, bucket, index); err != nil {
 				return nil, err
 			}
 		}
 	}
 	principal := s.tables.PrincipalFor(r)
-	if !s3tables.CheckPermissionWithContext(action, principal, record.OwnerAccountID, policy, resourceARN,
+	if !s.checkPermission(r, action, principal, record.OwnerAccountID, policy, resourceARN,
 		s.policyContext(r, bucket, bucketTags, resourceTags, req)) {
 		return nil, &s3tables.S3TablesError{Type: s3tables.ErrCodeAccessDenied, Message: "access denied"}
 	}
@@ -1154,47 +1158,90 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) {
 		catalogErr(w, err)
 		return
 	}
-	// Only a caller who passed the create check may learn an index exists.
+	metadata := map[string]interface{}{
+		"dimension":      fmt.Sprint(req.Dimension),
+		"distanceMetric": strings.ToLower(req.DistanceMetric),
+		"createNonce":    uuid.NewString(),
+		"initializing":   "true",
+	}
+	var nonFilterable []string
+	if req.MetadataConfiguration != nil {
+		nonFilterable = slices.Clone(req.MetadataConfiguration.NonFilterableMetadataKeys)
+		keys, _ := json.Marshal(nonFilterable)
+		metadata["nonFilterableMetadataKeys"] = string(keys)
+	}
 	var existing map[string]interface{}
 	getErr := s.internalCatalog(r.Context(), "GetTable", &s3tables.GetTableRequest{
 		TableBucketARN: s.s3tablesArn(arns.bucket),
 		Namespace:      []string{indexNamespace},
 		Name:           arns.index,
 	}, &existing)
+	created := false
+	versionToken := ""
 	switch {
 	case getErr == nil:
-		writeError(w, http.StatusConflict, "ConflictException",
-			fmt.Sprintf("index %s already exists", arns.index))
-		return
-	case catalogErrCode(getErr) != s3tables.ErrCodeNoSuchTable:
-		catalogErr(w, getErr)
-		return
-	}
-	metadata := map[string]string{
-		"dimension":      fmt.Sprint(req.Dimension),
-		"distanceMetric": strings.ToLower(req.DistanceMetric),
-		// The nonce marks the record this request created, so a rollback after
-		// an engine failure can never erase a pre-existing index.
-		"createNonce": uuid.NewString(),
-	}
-	var nonFilterable []string
-	if req.MetadataConfiguration != nil {
-		nonFilterable = req.MetadataConfiguration.NonFilterableMetadataKeys
-		if keys, err := json.Marshal(nonFilterable); err == nil {
-			metadata["nonFilterableMetadataKeys"] = string(keys)
+		index := indexFromRecord(s, arns.bucket, arns.index, existing)
+		storedMetric, _ := distanceMetric(index.DistanceMetric)
+		var storedKeys []string
+		if index.MetadataConfiguration != nil {
+			storedKeys = slices.Clone(index.MetadataConfiguration.NonFilterableMetadataKeys)
 		}
-	}
-	metadataJSON, _ := json.Marshal(metadata)
-	err = s.internalCatalog(bound, "CreateTable", &s3tables.CreateTableRequest{
-		TableBucketARN: s.s3tablesArn(arns.bucket),
-		Namespace:      []string{indexNamespace},
-		Name:           arns.index,
-		Format:         s3tables.FormatLance,
-		Metadata:       &s3tables.TableMetadata{FullMetadata: metadataJSON},
-		Tags:           req.Tags,
-	}, nil)
-	if err != nil {
-		catalogErr(w, err)
+		slices.Sort(storedKeys)
+		slices.Sort(nonFilterable)
+		meta, _ := existing["metadata"].(map[string]interface{})
+		full, _ := meta["fullMetadata"].(map[string]interface{})
+		if full["initializing"] != "true" || index.Dimension != req.Dimension ||
+			storedMetric != metric ||
+			!slices.Equal(storedKeys, nonFilterable) {
+			writeError(w, http.StatusConflict, "ConflictException", fmt.Sprintf("index %s already exists", arns.index))
+			return
+		}
+		metadata = full
+		metadataJSON, _ := json.Marshal(metadata)
+		var response s3tables.UpdateTableResponse
+		if err := s.internalCatalog(bound, "UpdateTable", &s3tables.UpdateTableRequest{
+			TableBucketARN: s.s3tablesArn(arns.bucket),
+			Namespace:      []string{indexNamespace},
+			Name:           arns.index,
+			VersionToken:   recordVersionToken(existing),
+			Metadata:       &s3tables.TableMetadata{FullMetadata: metadataJSON},
+		}, &response); err != nil {
+			catalogErr(w, err)
+			return
+		}
+		versionToken = response.VersionToken
+	case catalogErrCode(getErr) == s3tables.ErrCodeNoSuchTable:
+		metadataJSON, _ := json.Marshal(metadata)
+		var response s3tables.CreateTableResponse
+		err = s.internalCatalog(bound, "CreateTable", &s3tables.CreateTableRequest{
+			TableBucketARN: s.s3tablesArn(arns.bucket),
+			Namespace:      []string{indexNamespace},
+			Name:           arns.index,
+			Format:         s3tables.FormatLance,
+			Metadata:       &s3tables.TableMetadata{FullMetadata: metadataJSON},
+			Tags:           req.Tags,
+		}, &response)
+		if err != nil {
+			catalogErr(w, err)
+			return
+		}
+		var record map[string]interface{}
+		if err := s.internalCatalog(bound, "GetTable", &s3tables.GetTableRequest{
+			TableBucketARN: s.s3tablesArn(arns.bucket),
+			Namespace:      []string{indexNamespace},
+			Name:           arns.index,
+		}, &record); err != nil {
+			catalogErr(w, err)
+			return
+		}
+		if recordNonce(record) != metadata["createNonce"] || recordVersionToken(record) != response.VersionToken {
+			writeError(w, http.StatusConflict, "ConflictException", "index changed during creation; retry the request")
+			return
+		}
+		created = true
+		versionToken = response.VersionToken
+	default:
+		catalogErr(w, getErr)
 		return
 	}
 	_, err = s.vectors.CreateIndex(r.Context(), &vector_pb.CreateIndexRequest{
@@ -1204,17 +1251,27 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) {
 		NonFilterableMetadataKeys: nonFilterable,
 	})
 	if err != nil {
-		// Roll the index record back so a half-created index does not linger.
-		// The delete runs past a client disconnect and only removes the record
-		// this request created: the nonce must match, and the versionToken
-		// compare-and-swap keeps a racing create from being erased.
+		if !created {
+			vectorErr(w, r, arns, err)
+			return
+		}
+		if !errors.Is(err, errNoVectorWorker) {
+			switch status.Code(err) {
+			case codes.InvalidArgument, codes.PermissionDenied, codes.Unauthenticated, codes.FailedPrecondition, codes.AlreadyExists, codes.NotFound, codes.OutOfRange, codes.Unimplemented:
+			default:
+				vectorErr(w, r, arns, err)
+				return
+			}
+		}
+		// Only definite failures roll back; an unknown outcome may have committed.
+		// The nonce and version token protect a concurrent replacement.
 		ctx := context.WithoutCancel(r.Context())
 		var rec map[string]interface{}
 		if gerr := s.internalCatalog(ctx, "GetTable", &s3tables.GetTableRequest{
 			TableBucketARN: s.s3tablesArn(arns.bucket),
 			Namespace:      []string{indexNamespace},
 			Name:           arns.index,
-		}, &rec); gerr == nil && recordNonce(rec) == metadata["createNonce"] {
+		}, &rec); gerr == nil && recordNonce(rec) == metadata["createNonce"] && recordVersionToken(rec) == versionToken {
 			var delResp map[string]interface{}
 			if derr := s.internalCatalog(ctx, "DeleteTable", &s3tables.DeleteTableRequest{
 				TableBucketARN: s.s3tablesArn(arns.bucket),
@@ -1230,6 +1287,18 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) {
 		vectorErr(w, r, arns, err)
 		return
 	}
+	delete(metadata, "initializing")
+	metadataJSON, _ := json.Marshal(metadata)
+	if err := s.internalCatalog(r.Context(), "UpdateTable", &s3tables.UpdateTableRequest{
+		TableBucketARN: s.s3tablesArn(arns.bucket),
+		Namespace:      []string{indexNamespace},
+		Name:           arns.index,
+		VersionToken:   versionToken,
+		Metadata:       &s3tables.TableMetadata{FullMetadata: metadataJSON},
+	}, nil); err != nil {
+		catalogErr(w, err)
+		return
+	}
 	writeJSON(w, http.StatusOK, &CreateIndexResponse{IndexArn: s.indexArn(arns.bucket, arns.index)})
 }
 

weed/s3api/s3vectors/iam.go

@@ -0,0 +1,26 @@
+package s3vectors
+
+import (
+	"net/http"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+)
+
+type IAMAuthorizer interface {
+	EvaluateIAMPolicy(r *http.Request, action, resource string, policyContext *s3tables.PolicyContext) (allowed, denied bool)
+}
+
+func (s *Server) checkPermission(r *http.Request, action, principal, owner, policy, resource string, ctx *s3tables.PolicyContext) bool {
+	if authorizer, ok := s.authenticator.(IAMAuthorizer); ok {
+		allowed, denied := authorizer.EvaluateIAMPolicy(r, action, resource, ctx)
+		if denied {
+			return false
+		}
+		if allowed {
+			copyContext := *ctx
+			copyContext.IdentityActions = append(append([]string{}, ctx.IdentityActions...), action)
+			ctx = &copyContext
+		}
+	}
+	return s3tables.CheckPermissionWithContext(action, principal, owner, policy, resource, ctx)
+}

weed/s3api/s3vectors/iam_test.go

@@ -0,0 +1,54 @@
+package s3vectors
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
+)
+
+type iamAuthenticator struct {
+	fakeAuthenticator
+	allowed, denied bool
+}
+
+func (a *iamAuthenticator) EvaluateIAMPolicy(*http.Request, string, string, *s3tables.PolicyContext) (bool, bool) {
+	return a.allowed, a.denied
+}
+
+func TestIAMGrantRespectsVectorResourceDenies(t *testing.T) {
+	auth := &iamAuthenticator{allowed: true}
+	server := &Server{authenticator: auth}
+	request := httptest.NewRequest("POST", "/QueryVectors", nil)
+	const action = "s3vectors:QueryVectors"
+	const resource = "arn:aws:s3vectors:us-east-1:000000000000:bucket/embeddings/index/docs"
+	ctx := &s3tables.PolicyContext{IdentityActions: []string{}}
+	if !server.checkPermission(request, action, "reader", "owner", "", resource, ctx) {
+		t.Fatal("IAM grant denied")
+	}
+	deny := `{"Statement":[{"Effect":"Deny","Principal":"*","Action":"s3vectors:*","Resource":"*"}]}`
+	if server.checkPermission(request, action, "reader", "owner", deny, resource, ctx) {
+		t.Fatal("resource deny was overridden by IAM")
+	}
+	auth.denied = true
+	if server.checkPermission(request, action, "owner", "owner", "", resource, ctx) {
+		t.Fatal("identity deny was overridden by ownership")
+	}
+}
+
+type requestTagAuthenticator struct{ fakeAuthenticator }
+
+func (*requestTagAuthenticator) EvaluateIAMPolicy(_ *http.Request, _, _ string, ctx *s3tables.PolicyContext) (bool, bool) {
+	return true, ctx.RequestTags["classification"] == "forbidden"
+}
+
+func TestIAMCreateBucketReceivesRequestTags(t *testing.T) {
+	fs := s3tablestest.Start(t)
+	server := NewServer(s3tables.NewManagerClient(fs.Client), &requestTagAuthenticator{}, nil)
+	response := postJSONAs(t, "", server.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "blocked", Tags: map[string]string{"classification": "forbidden"}})
+	if response.Code != http.StatusForbidden {
+		t.Fatalf("creation tag deny bypassed: %d %s", response.Code, response.Body)
+	}
+}

weed/s3api/s3vectors/operations.go

@@ -0,0 +1,12 @@
+package s3vectors
+
+import "sort"
+
+func Operations() []string {
+	operations := make([]string, 0, len(handlers))
+	for operation := range handlers {
+		operations = append(operations, operation)
+	}
+	sort.Strings(operations)
+	return operations
+}

weed/shell/commands.go

@@ -149,6 +149,10 @@ func (ce *CommandEnv) checkDirectory(path string) error {
 
 var _ = filer_pb.FilerClient(&CommandEnv{})
 
+func (ce *CommandEnv) GrpcDialOption() grpc.DialOption {
+	return ce.option.GrpcDialOption
+}
+
 func (ce *CommandEnv) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
 
 	return pb.WithGrpcFilerClient(streamingMode, 0, ce.FilerAddress(), ce.option.GrpcDialOption, fn)