s3vectors: validate index ARNs and expand live integration coverage (#12222)

* test: align vector bucket policy coverage with vector actions

* s3vectors: validate ARN service and resource shape before resolving indexes

* test: exercise signed vector bucket policies and catalog lifecycle

* ci: require the expanded vector integration matrix and all gateway tests

* test: reject index ARNs in bucket-level API requests

* s3vectors: scope ARN resolution to the gateway namespace

* s3vectors: reject name fields that conflict with resolved ARNs

bucketARNs and resolveIndexARNs already reject an indexArn that names a
different bucket than vectorBucketArn, but a vectorBucketName or
indexName disagreeing with the ARNs was silently dropped. Validate the
plain names against the resolved ARN values the same way.

* test: cover conflicting vector names and ARNs in live requests

---------

Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

test/vectors/README.md

@@ -3,10 +3,12 @@
 Run `python3 -m pip install -r requirements.txt`, then `make test`.
 
 The suite starts a master, volume, filer, authenticated S3/Lance gateway, and
-vector engine on temporary ports. The same 12 API tests run with both Go and
+vector engine on temporary ports. The same 17 API tests run with both Go and
 Rust volume servers. Coverage includes bucket/index lifecycle, bucket policies,
-empty results, ARN addressing, metadata, filters, cosine distance, atomic upserts,
-pagination, segmented listing, authentication, and writer loss.
+signed policy grants and denies, encryption settings, empty results, ARN validation,
+metadata, filters, cosine distance, atomic upserts,
+catalog/vector pagination, index deletion and recreation, segmented listing,
+authentication, and writer loss.
 
 Server logs and data stay in pytest's temporary directory for diagnosis. Every
 child process stops on exit. To select a volume server, set

test/vectors/conftest.py

@@ -155,3 +155,13 @@ def vectors(cluster):
     return boto3.client("s3vectors", endpoint_url=cluster["endpoint"],
                         aws_access_key_id="test", aws_secret_access_key="test", region_name="us-east-1",
                         config=Config(connect_timeout=3, read_timeout=20, retries={"max_attempts": 0}))
+
+
+@pytest.fixture(scope="module")
+def vector_clients(vectors):
+    return {
+        name: boto3.client("s3vectors", endpoint_url=vectors.meta.endpoint_url,
+                           aws_access_key_id=name, aws_secret_access_key=name + "-secret",
+                           region_name="us-east-1", config=vectors.meta.config)
+        for name in ("reader", "writer")
+    }

test/vectors/s3.json

@@ -15,6 +15,30 @@
         "Tagging",
         "Write"
       ]
+    },
+    {
+      "name": "reader",
+      "credentials": [
+        {
+          "accessKey": "reader",
+          "secretKey": "reader-secret"
+        }
+      ],
+      "actions": [
+        "Read"
+      ]
+    },
+    {
+      "name": "writer",
+      "credentials": [
+        {
+          "accessKey": "writer",
+          "secretKey": "writer-secret"
+        }
+      ],
+      "actions": [
+        "Write"
+      ]
     }
   ]
 }

test/vectors/s3vectors_test.py

@@ -85,6 +85,10 @@ def test_vector_bucket_policy(vectors):
                 ],
             }
         )
+        invalid = json.loads(policy)
+        invalid["Statement"][0]["Action"] = "s3tables:*"
+        expect_error("ValidationException", vectors.put_vector_bucket_policy,
+                     vectorBucketName=BUCKET, policy=json.dumps(invalid))
         vectors.put_vector_bucket_policy(vectorBucketName=BUCKET, policy=policy)
         got = vectors.get_vector_bucket_policy(vectorBucketName=BUCKET)["policy"]
         assert json.loads(got) == json.loads(policy)
@@ -217,6 +221,141 @@ def test_delete_bucket_conflict(vectors, index):
     assert vectors.get_index(**index)["index"]["dimension"] == 4
 
 
+def test_vector_policy_permissions(vectors, vector_clients, index):
+    reader, writer = vector_clients["reader"], vector_clients["writer"]
+    arn = vectors.get_index(**index)["index"]["indexArn"]
+    row = {"key": "policy-row", "data": {"float32": [1, 0, 0, 0]}, "metadata": {"owner": "writer"}}
+    expect_error("AccessDeniedException", writer.put_vectors, **index, vectors=[row])
+    expect_error("AccessDeniedException", reader.get_vectors, **index, keys=[row["key"]])
+    statements = [
+        {"Effect": "Allow", "Principal": {"AWS": "writer"},
+         "Action": "s3vectors:PutVectors", "Resource": arn},
+        {"Effect": "Allow", "Principal": {"AWS": "reader"},
+         "Action": ["s3vectors:GetVectors", "s3vectors:QueryVectors"], "Resource": arn},
+    ]
+
+    def put_policy():
+        vectors.put_vector_bucket_policy(vectorBucketName=index["vectorBucketName"],
+                                         policy=json.dumps({"Version": "2012-10-17", "Statement": statements}))
+
+    put_policy()
+    writer.put_vectors(**index, vectors=[row])
+    assert reader.get_vectors(**index, keys=[row["key"]], returnData=True, returnMetadata=True)["vectors"] == [row]
+    assert reader.query_vectors(**index, queryVector=row["data"], topK=1)["vectors"][0]["key"] == row["key"]
+    expect_error("AccessDeniedException", reader.put_vectors, **index, vectors=[row])
+    expect_error("AccessDeniedException", writer.get_vectors, **index, keys=[row["key"]])
+    expect_error("AccessDeniedException", reader.delete_index, **index)
+    expect_error("AccessDeniedException", reader.list_vectors, **index)
+    other = dict(index, indexName="other-index")
+    vectors.create_index(**other, dataType="float32", dimension=4, distanceMetric="euclidean")
+    try:
+        expect_error("AccessDeniedException", reader.get_vectors, **other, keys=[row["key"]])
+        expect_error("AccessDeniedException", writer.put_vectors, **other, vectors=[row])
+        statements.append({"Effect": "Deny", "Principal": {"AWS": "writer"},
+                           "Action": "s3vectors:PutVectors", "Resource": arn})
+        put_policy()
+        expect_error("AccessDeniedException", writer.put_vectors, **index, vectors=[row])
+        assert reader.get_vectors(**index, keys=[row["key"]])["vectors"][0]["key"] == row["key"]
+        vectors.delete_vector_bucket_policy(vectorBucketName=index["vectorBucketName"])
+        expect_error("AccessDeniedException", reader.get_vectors, **index, keys=[row["key"]])
+        expect_error("AccessDeniedException", writer.put_vectors, **index, vectors=[row])
+        vectors.put_vectors(**index, vectors=[row])
+    finally:
+        vectors.delete_index(**other)
+
+
+def test_vector_bucket_encryption(vectors):
+    bucket = "encryption-" + uuid.uuid4().hex[:16]
+    config = {"sseType": "AES256"}
+    vectors.create_vector_bucket(vectorBucketName=bucket, encryptionConfiguration=config)
+    try:
+        assert vectors.get_vector_bucket(vectorBucketName=bucket)["vectorBucket"]["encryptionConfiguration"] == config
+        expect_error("ValidationException", vectors.create_vector_bucket, vectorBucketName=bucket + "-kms",
+                     encryptionConfiguration={"sseType": "aws:kms", "kmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/test"})
+        assert vectors.list_vector_buckets(prefix=bucket + "-kms")["vectorBuckets"] == []
+    finally:
+        vectors.delete_vector_bucket(vectorBucketName=bucket)
+
+
+def test_catalog_pagination(vectors, index):
+    prefix = "pages-" + uuid.uuid4().hex[:16]
+    buckets, indexes = [], []
+    try:
+        for i in range(3):
+            bucket = f"{prefix}-{i}"
+            vectors.create_vector_bucket(vectorBucketName=bucket)
+            buckets.append(bucket)
+        for name in ("idx-a", "idx-b"):
+            args = dict(index, indexName=name)
+            vectors.create_index(**args, dataType="float32", dimension=4, distanceMetric="euclidean")
+            indexes.append(args)
+        pages = list(vectors.get_paginator("list_vector_buckets").paginate(
+            prefix=prefix, PaginationConfig={"PageSize": 1}))
+        assert all(len(page["vectorBuckets"]) <= 1 for page in pages)
+        assert sorted(b["vectorBucketName"] for page in pages for b in page["vectorBuckets"]) == buckets
+        pages = list(vectors.get_paginator("list_indexes").paginate(
+            vectorBucketName=index["vectorBucketName"], prefix="idx", PaginationConfig={"PageSize": 1}))
+        assert all(len(page["indexes"]) <= 1 for page in pages)
+        assert sorted(i["indexName"] for page in pages for i in page["indexes"]) == [INDEX, "idx-a", "idx-b"]
+        expect_error("ValidationException", vectors.list_vector_buckets, maxResults=501)
+        expect_error("ValidationException", vectors.list_indexes, vectorBucketName=index["vectorBucketName"], maxResults=501)
+    finally:
+        for args in indexes:
+            vectors.delete_index(**args)
+        for bucket in buckets:
+            vectors.delete_vector_bucket(vectorBucketName=bucket)
+
+
+def test_delete_and_recreate_index(vectors, index):
+    vectors.put_vectors(**index, vectors=[{"key": "old", "data": {"float32": [1, 0, 0, 0]}}])
+    vectors.delete_index(**index)
+    expect_error("NotFoundException", vectors.get_index, **index)
+    vectors.create_index(**index, dataType="float32", dimension=2, distanceMetric="cosine")
+    assert vectors.list_vectors(**index)["vectors"] == []
+    row = {"key": "new", "data": {"float32": [1, 0]}}
+    vectors.put_vectors(**index, vectors=[row])
+    assert vectors.get_vectors(**index, keys=["old", "new"], returnData=True)["vectors"] == [row]
+    result = vectors.query_vectors(**index, queryVector=row["data"], topK=1, returnDistance=True)
+    assert result["distanceMetric"] == "cosine"
+    assert result["vectors"][0]["key"] == "new"
+    assert result["vectors"][0]["distance"] == pytest.approx(0, abs=1e-6)
+
+
+def test_index_arn_validation(vectors, index):
+    arn = vectors.get_index(**index)["index"]["indexArn"]
+    foreign = []
+    for field, value in ((1, "aws-cn"), (3, "eu-west-1"), (4, "999999999999")):
+        parts = arn.split(":", 5)
+        assert parts[field] != value
+        parts[field] = value
+        foreign.append(":".join(parts))
+    for invalid in (
+        *foreign,
+        arn.replace(":s3vectors:", ":s3tables:"),
+        arn.replace(":s3vectors:", ":s3:"),
+        arn + "/index/extra",
+        arn + "/",
+    ):
+        expect_error("ValidationException", vectors.get_index, indexArn=invalid)
+        expect_error("ValidationException", vectors.delete_index, indexArn=invalid)
+    expect_error("ValidationException", vectors.get_vector_bucket, vectorBucketArn=arn)
+    expect_error("ValidationException", vectors.list_indexes, vectorBucketArn=arn)
+    bucket_arn = arn.rsplit("/index/", 1)[0]
+    for request in (
+        {"vectorBucketName": "other-" + index["vectorBucketName"], "indexArn": arn},
+        {"indexName": "other-" + index["indexName"], "indexArn": arn},
+    ):
+        expect_error("ValidationException", vectors.get_index, **request)
+        expect_error("ValidationException", vectors.delete_index, **request)
+    for operation in (vectors.get_vector_bucket, vectors.list_indexes, vectors.delete_vector_bucket):
+        expect_error("ValidationException", operation, vectorBucketName="other-" + index["vectorBucketName"],
+                     vectorBucketArn=bucket_arn)
+    assert vectors.get_index(**index, indexArn=arn)["index"]["indexArn"] == arn
+    assert vectors.get_vector_bucket(vectorBucketName=index["vectorBucketName"],
+                                     vectorBucketArn=bucket_arn)["vectorBucket"]["vectorBucketArn"] == bucket_arn
+    assert vectors.get_index(**index)["index"]["indexArn"] == arn
+
+
 def test_invalid_credentials(cluster):
     client = boto3.client("s3vectors", endpoint_url=cluster["endpoint"], aws_access_key_id="unknown",
                           aws_secret_access_key="unknown", region_name="us-east-1")