vector: integrate engine health, admin management, and local deployment (#12517)

* vector: expose request metrics and registration-aware health endpoints

* vector: report immutable read cache and storage fetch metrics

* docs: describe vector engine health, metrics, and maintenance

* vector: count canceled requests when their handler is dropped

* vector: account for uncached large immutable storage reads

* vector: reliably release the stalled credential reader in tests

* vector: observe large reads as their streamed body is consumed

* vector: bound heartbeats on established master registration streams

* test: exercise hardened vector maintenance on Go and Rust volumes

* vector: register engines in their configured filer group

* vector: make retried index creation preserve compatible datasets

* vector: allow registration to advertise a separate address

* test: release stalled credential readers without blocking FIFO cleanup

* test: allow hardened vector coverage in a named filer group

* vector: discover engines in the same filer group as their gateway

* vector: fall back when the advertised address is empty

* admin: expose vector readiness, maintenance, and management tools (#12520)

* s3vectors: share vector discovery with trusted management clients

* admin: show vector engine readiness and recent maintenance activity

* admin: suggest S3 Vectors IAM actions from the supported operations

* admin: manage vector indexes, policies, tags, encryption, and query data

* admin: validate SSE-S3 support before creating encrypted vector buckets

* admin: discover vector engines in their registration group

* admin: probe vector engines concurrently with independent time budgets

* admin: verify encryption capability on the remote S3 gateway

* admin: defer discovery until after the HTTP server can start

* admin: distinguish unknown vector health services from failed engines

* admin: count workers that can execute vector index maintenance

* admin: allow read-only users to query vectors and inspect tags

* vector: scope gateway and admin discovery to the configured filer group

* admin: refresh policy state after mutations and serialize policy controls

* vector: drain active RPCs before closing replaced discovery connections

* vector: respect request cancellation while waiting for discovery

* vector: prefer another registered engine after a transport failure

* admin: check all masters before reporting no registered vector engines

* iam: honor attached vector policies without broadening legacy grants

* vector: allow short-lived discovery clients to close their connections

* vector: refresh cached master addresses for each discovery lookup

* iam: enforce vector bucket tag conditions in attached policies

* iam: authorize vector bucket creation with the requested tags

* test: exercise vector IAM grants and tag denies on Go and Rust volumes

* admin: reject authenticated vector mutations without an admin role

* test: run vector IAM assertions before the shared engine is stopped

* vector: add native mini startup, shell tools, and a runnable Compose example (#12522)

* s3vectors: share vector discovery with trusted management clients

* admin: show vector engine readiness and recent maintenance activity

* admin: suggest S3 Vectors IAM actions from the supported operations

* admin: manage vector indexes, policies, tags, encryption, and query data

* admin: validate SSE-S3 support before creating encrypted vector buckets

* admin: discover vector engines in their registration group

* admin: probe vector engines concurrently with independent time budgets

* admin: verify encryption capability on the remote S3 gateway

* admin: defer discovery until after the HTTP server can start

* admin: distinguish unknown vector health services from failed engines

* admin: count workers that can execute vector index maintenance

* admin: allow read-only users to query vectors and inspect tags

* vector: scope gateway and admin discovery to the configured filer group

* admin: refresh policy state after mutations and serialize policy controls

* vector: drain active RPCs before closing replaced discovery connections

* vector: respect request cancellation while waiting for discovery

* vector: prefer another registered engine after a transport failure

* admin: check all masters before reporting no registered vector engines

* iam: honor attached vector policies without broadening legacy grants

* vector: allow short-lived discovery clients to close their connections

* vector: refresh cached master addresses for each discovery lookup

* iam: enforce vector bucket tag conditions in attached policies

* iam: authorize vector bucket creation with the requested tags

* test: exercise vector IAM grants and tag denies on Go and Rust volumes

* admin: reject authenticated vector mutations without an admin role

* test: run vector IAM assertions before the shared engine is stopped

* mini: optionally start and supervise the vector engine

* mini: bootstrap vector buckets with existing Lance catalog helpers

* shell: manage vector buckets, indexes, and vector data

* docs: provide a runnable vector and maintenance worker Compose stack

* shell: scope vector discovery to the session and close its connections

* mini: honor the bind address for vector and metrics listeners

* mini: reserve vector ports before assigning gRPC and compatibility listeners

* mini: wait for vector storage setup and master registration

* mini: create requested vector buckets without requiring the Lance HTTP listener

* shell: refresh master addresses during vector discovery

* mini: resolve sibling vector executables with platform extensions

* vector: preserve uncertain creates and fix mini listener allocation (#12525)

* vector: retain catalog records when index creation has an unknown outcome

* mini: keep chosen listener ports reserved throughout allocation

* mini: resolve hostname binds before starting vector listeners

* vector: fix integration test imports and recover interrupted index creation (#12533)

* shell: register vector operations from the command package

* s3vectors: resume interrupted index initialization safely

* mini: exclude disabled WebDAV from listener allocation

* s3vectors: retain uncertain creates across refused retries

* s3vectors: claim pending indexes before retrying initialization

* s3vectors: compare resolved distance metrics during recovery

* s3vectors: preserve metadata keys when retrying initialization

* s3vectors: verify ownership before initializing new catalog records

test/s3vectors/conftest.py

@@ -123,7 +123,7 @@ def write_tls(directory):
             "-out", "outsider.pem", "-days", "2", "-extfile", "ext.cnf")
     cert, key, ca = (f'{key} = {json.dumps(str(directory / name), ensure_ascii=False)}\n' for key, name in (("cert", "cert.pem"), ("key", "key.pem"), ("ca", "ca.pem")))
     pem = cert + key + ca
-    grpc = "".join(f"[grpc.{component}]\n{cert}{key}\n" for component in ("master", "volume", "filer", "s3"))
+    grpc = "".join(f"[grpc.{component}]\n{cert}{key}\n" for component in ("master", "volume", "filer", "s3", "admin", "worker"))
     # Clients (the gateway, the engine) accept only servers named localhost;
     # the engine serves only clients named localhost.
     grpc += f"[grpc.client]\n{cert}{key}allowed_server_commonNames = \"localhost\"\n"
@@ -248,12 +248,12 @@ def run_cluster(param, tmp_path_factory, identities):
     with contextlib.ExitStack() as stack:
         with contextlib.ExitStack() as reservations:
             ports = []
-            for _ in range(14):
+            for _ in range(15):
                 sock = reservations.enter_context(socket.socket())
                 sock.bind(("127.0.0.1", 0))
                 ports.append(sock.getsockname()[1])
         (master, master_grpc, vol, vol_grpc, filer, filer_grpc, s3, s3_grpc, lance, vector, worker1, worker2,
-         admin, admin_grpc) = ports
+         admin, admin_grpc, vector_metrics) = ports
         master_addr = f"127.0.0.1:{master}.{master_grpc}"
         filer_addr = f"127.0.0.1:{filer}.{filer_grpc}"
 
@@ -304,10 +304,11 @@ def run_cluster(param, tmp_path_factory, identities):
         # VECTOR_ENCRYPT_VOLUMES=1 turns on the filer's volume encryption, which
         # the gateway, the engine and the worker all follow: every chunk they
         # store is encrypted, and direct reads must undo it.
+        filer_group = os.environ.get("VECTOR_FILER_GROUP", "")
         encrypt = ["-encryptVolumeData"] if os.environ.get("VECTOR_ENCRYPT_VOLUMES") else []
         start("filer", [weed, "filer", f"-master={master_addr}", "-ip=127.0.0.1",
                         "-ip.bind=127.0.0.1", f"-port={filer}", f"-port.grpc={filer_grpc}",
-                        f"-defaultStoreDir={directory / 'filer'}", *encrypt])
+                        f"-defaultStoreDir={directory / 'filer'}", f"-filerGroup={filer_group}", *encrypt])
         wait_for(lambda: listening(filer_grpc), processes, "filer not ready")
         endpoint = f"http://127.0.0.1:{s3}"
         # weed s3 turns AWS_ACCESS_KEY_ID into an admin identity.
@@ -325,8 +326,12 @@ def run_cluster(param, tmp_path_factory, identities):
         # The engine keeps tables on the filers and volume servers itself; it
         # has no S3 credentials to be given, and encrypts its chunks for the
         # volume servers as the gateway does.
-        start("vector", [engine, "--masters", master_addr, "--listen", f"127.0.0.1:{vector}"])
+        start("vector", [engine, "--masters", master_addr, "--listen", f"127.0.0.1:{vector}", "--metrics-port", str(vector_metrics), "--filer-group", filer_group])
         wait_for(lambda: listening(vector), processes, "vector engine not ready")
+        def vector_ready():
+            with urllib.request.urlopen(f"http://127.0.0.1:{vector_metrics}/ready", timeout=1) as response:
+                return response.status == 200
+        wait_for(vector_ready, processes, "vector engine did not register with master")
         if os.environ.get("VECTOR_TLS"):
             wait_for(lambda: listening(s3_grpc), processes, "s3 gRPC not ready")
             for name, port in (("master", master_grpc), ("volume", vol_grpc), ("filer", filer_grpc),
@@ -335,7 +340,7 @@ def run_cluster(param, tmp_path_factory, identities):
             probe_grpc_tls("engine", vector, directory, outsider=True)
 
         wait_for(
-            lambda: f"Cluster: vector node 127.0.0.1:{vector} added to group ''"
+            lambda: f"Cluster: vector node 127.0.0.1:{vector} added to group '{filer_group}'"
             in (directory / "master.log").read_text(),
             processes,
             "vector engine never registered with the master",
@@ -358,18 +363,22 @@ def run_cluster(param, tmp_path_factory, identities):
         if lance_worker:
             (directory / "admin").mkdir()
             admin_url = f"http://127.0.0.1:{admin}"
-            start("admin", [weed, "admin", "-ip=127.0.0.1", f"-port={admin}", f"-port.grpc={admin_grpc}",
+            start("admin", [weed, "admin", f"-filerGroup={filer_group}", "-ip=127.0.0.1", f"-port={admin}", f"-port.grpc={admin_grpc}",
                             "-port.legacy=0", f"-master={master_addr}", f"-dataDir={directory / 'admin'}"])
             wait_for(lambda: http_ready(f"{admin_url}/api/plugin/job-types"), processes, "admin not ready")
+            tls_args = []
+            if os.environ.get("VECTOR_TLS"):
+                tls_args = ["--tls-ca", str(directory / "ca.pem"), "--tls-cert", str(directory / "cert.pem"), "--tls-key", str(directory / "key.pem"), "--tls-server-name", "localhost"]
             start("lance-worker", [lance_worker, "--admin", f"127.0.0.1:{admin}.{admin_grpc}",
                                    "--namespace", f"http://127.0.0.1:{lance}", "--id", "vectors-test-worker",
-                                   "--masters", master_addr])
+                                   "--masters", master_addr, "--filer-group", filer_group, *tls_args])
             wait_for(lambda: "lance_vector_index" in
                      [jt["job_type"] for jt in json.load(urllib.request.urlopen(f"{admin_url}/api/plugin/job-types", timeout=2))],
                      processes, "the Lance worker never registered with admin")
 
         yield {"endpoint": endpoint, "mode": mode, "processes": processes, "directory": directory, "filer": filer,
                "masters": master_addr, "filer_signing_key": filer_signing_key,
+               "vector_metrics": f"http://127.0.0.1:{vector_metrics}",
                "catalog": f"http://127.0.0.1:{lance}", "admin": admin_url, "env": process_env()}
 
 

test/s3vectors/s3.json

@@ -39,6 +39,24 @@
       "actions": [
         "Write"
       ]
+    },
+    {
+      "name": "policy-reader",
+      "credentials": [
+        {
+          "accessKey": "policy-reader",
+          "secretKey": "policy-reader-secret"
+        }
+      ],
+      "policyNames": [
+        "vector-policy"
+      ]
+    }
+  ],
+  "policies": [
+    {
+      "name": "vector-policy",
+      "content": "{\"Version\": \"2012-10-17\", \"Statement\": [{\"Effect\": \"Allow\", \"Action\": [\"s3vectors:GetVectors\", \"s3vectors:QueryVectors\"], \"Resource\": \"arn:aws:s3vectors:*:*:bucket/vector-iam/index/docs\"}, {\"Effect\": \"Allow\", \"Action\": \"s3vectors:CreateVectorBucket\", \"Resource\": \"*\"}, {\"Effect\": \"Allow\", \"Action\": \"s3vectors:TagResource\", \"Resource\": \"arn:aws:s3vectors:*:*:bucket/vector-iam-*\"}, {\"Effect\": \"Deny\", \"Action\": \"s3vectors:QueryVectors\", \"Resource\": \"*\", \"Condition\": {\"StringEquals\": {\"aws:ResourceTag/classification\": \"private\"}}}, {\"Effect\": \"Deny\", \"Action\": \"s3vectors:QueryVectors\", \"Resource\": \"*\", \"Condition\": {\"StringEquals\": {\"s3vectors:VectorBucketTag/environment\": \"production\"}}}, {\"Effect\": \"Deny\", \"Action\": \"s3vectors:CreateVectorBucket\", \"Resource\": \"*\", \"Condition\": {\"StringEquals\": {\"aws:RequestTag/classification\": \"forbidden\"}}}, {\"Effect\": \"Deny\", \"Action\": \"s3vectors:CreateVectorBucket\", \"Resource\": \"*\", \"Condition\": {\"ForAnyValue:StringEquals\": {\"aws:TagKeys\": \"forbidden-key\"}}}]}"
     }
   ]
 }

test/s3vectors/s3vectors_test.py

@@ -1,3 +1,4 @@
+import urllib.request
 """Run the S3 Vectors API against Go/Rust volumes and single/distributed queries."""
 
 import contextlib
@@ -150,7 +151,7 @@ def run_cli(cluster, *args):
     from pathlib import Path
     cli = Path(os.environ.get("VECTOR_CLI_BINARY", Path(__file__).resolve().parent / "weed-vector-cli")).resolve()
     command = [str(cli), "--namespace", cluster["catalog"], "--masters", cluster["masters"],
-               "--filer-signing-key", cluster["filer_signing_key"], *args]
+               "--filer-signing-key", cluster["filer_signing_key"], "--filer-group", os.environ.get("VECTOR_FILER_GROUP", ""), *args]
     return json.loads(subprocess.run(command, check=True, capture_output=True, text=True,
                                      cwd=cluster["directory"], env=cluster["env"]).stdout)
 
@@ -711,6 +712,43 @@ def test_vector_index_job(vectors, cluster):
     assert found >= 180, f"recall after retrain {found}/200"
 
 
+def test_attached_iam_vector_policies(vectors, cluster):
+    client = boto3.client("s3vectors", endpoint_url=cluster["endpoint"], region_name="us-east-1",
+                          aws_access_key_id="policy-reader", aws_secret_access_key="policy-reader-secret",
+                          config=vectors.meta.config)
+    bucket = "vector-iam"
+    vectors.create_vector_bucket(vectorBucketName=bucket)
+    target = {"vectorBucketName": bucket, "indexName": "docs"}
+    vectors.create_index(**target, dataType="float32", dimension=3, distanceMetric="cosine")
+    index_arn = vectors.get_index(**target)["index"]["indexArn"]
+    bucket_arn = vectors.get_vector_bucket(vectorBucketName=bucket)["vectorBucket"]["vectorBucketArn"]
+    query = {**target, "queryVector": {"float32": [1, 0, 0]}, "topK": 1}
+    try:
+        assert client.query_vectors(**query)["vectors"] == []
+        assert client.get_vectors(**target, keys=["missing"])["vectors"] == []
+        expect_error("AccessDeniedException", client.put_vectors, **target,
+                     vectors=[{"key": "blocked", "data": {"float32": [1, 0, 0]}}])
+        vectors.tag_resource(resourceArn=index_arn, tags={"classification": "private"})
+        expect_error("AccessDeniedException", client.query_vectors, **query)
+        vectors.untag_resource(resourceArn=index_arn, tagKeys=["classification"])
+        assert client.query_vectors(**query)["vectors"] == []
+        vectors.tag_resource(resourceArn=bucket_arn, tags={"environment": "production"})
+        expect_error("AccessDeniedException", client.query_vectors, **query)
+        vectors.untag_resource(resourceArn=bucket_arn, tagKeys=["environment"])
+        assert client.query_vectors(**query)["vectors"] == []
+        vectors.put_vector_bucket_policy(vectorBucketName=bucket, policy=json.dumps({"Version": "2012-10-17", "Statement": [
+            {"Effect": "Deny", "Principal": "*", "Action": "s3vectors:QueryVectors", "Resource": index_arn}]}))
+        expect_error("AccessDeniedException", client.query_vectors, **query)
+        expect_error("AccessDeniedException", client.create_vector_bucket,
+                     vectorBucketName="vector-iam-rejected", tags={"classification": "forbidden"})
+        expect_error("AccessDeniedException", client.create_vector_bucket,
+                     vectorBucketName="vector-iam-key-denied", tags={"forbidden-key": "value"})
+    finally:
+        vectors.delete_vector_bucket_policy(vectorBucketName=bucket)
+        vectors.delete_index(**target)
+        vectors.delete_vector_bucket(vectorBucketName=bucket)
+
+
 def test_invalid_credentials(cluster):
     client = boto3.client("s3vectors", endpoint_url=cluster["endpoint"], aws_access_key_id="unknown",
                           aws_secret_access_key="unknown", region_name="us-east-1")
@@ -719,6 +757,18 @@ def test_invalid_credentials(cluster):
     assert exc.value.response["ResponseMetadata"]["HTTPStatusCode"] == 403
 
 
+def test_vector_engine_observability(vectors, index, cluster):
+    vectors.query_vectors(**index, queryVector={"float32": [1, 0, 0, 0]}, topK=1)
+    with urllib.request.urlopen(cluster["vector_metrics"] + "/ready", timeout=5) as response:
+        assert response.status == 200
+    with urllib.request.urlopen(cluster["vector_metrics"] + "/metrics", timeout=5) as response:
+        metrics = response.read().decode()
+    assert "SeaweedFS_vector_connected 1" in metrics
+    assert any(line.startswith("SeaweedFS_vector_request_total{") and 'type="QueryVectors"' in line
+               and 'code="OK"' in line for line in metrics.splitlines())
+    assert "SeaweedFS_vector_request_seconds_bucket" in metrics
+
+
 def test_query_without_writer(vectors, index, cluster):
     from conftest import stop
 
@@ -753,3 +803,5 @@ def test_engine_without_credentials(open_cluster):
     hits = client.query_vectors(**index, queryVector={"float32": [1, 0]}, topK=1)["vectors"]
     assert [h["key"] for h in hits] == ["a"]
 
+
+