diff --git a/.github/workflows/rust-vector-tests.yml b/.github/workflows/rust-vector-tests.yml index fbe3e01a9a..0db11314de 100644 --- a/.github/workflows/rust-vector-tests.yml +++ b/.github/workflows/rust-vector-tests.yml @@ -131,7 +131,9 @@ jobs: test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } echo "PROTOC=$protoc" >> "$GITHUB_ENV" - name: Test Go vector gateway - run: go test ./weed/s3api/s3vectors ./weed/command -run 'Vector|Vectors' -count=1 + run: | + go test ./weed/s3api/s3vectors -count=1 + go test ./weed/command -run 'Vector|Vectors' -count=1 - name: Build integration binaries working-directory: test/vectors run: make build CARGO_PROFILE=dev @@ -149,10 +151,10 @@ jobs: from pathlib import Path import xml.etree.ElementTree as ET cases = ET.parse(Path(os.environ["RUNNER_TEMP"]) / "vector-results.xml").findall(".//testcase") - assert len(cases) >= 24, f"expected at least 24 cases, got {len(cases)}" + assert len(cases) >= 34, f"expected at least 34 cases, got {len(cases)}" assert not any(case.find("skipped") is not None for case in cases), "integration cases were skipped" for volume in ("go", "rust"): - assert sum(case.get("name", "").endswith(f"[{volume}-single]") for case in cases) >= 12, volume + assert sum(case.get("name", "").endswith(f"[{volume}-single]") for case in cases) >= 17, volume PYTHON - name: Upload integration diagnostics if: always() diff --git a/test/vectors/README.md b/test/vectors/README.md index a0efbea15f..4f78d718c5 100644 --- a/test/vectors/README.md +++ b/test/vectors/README.md @@ -3,10 +3,12 @@ Run `python3 -m pip install -r requirements.txt`, then `make test`. The suite starts a master, volume, filer, authenticated S3/Lance gateway, and -vector engine on temporary ports. The same 12 API tests run with both Go and +vector engine on temporary ports. The same 17 API tests run with both Go and Rust volume servers. Coverage includes bucket/index lifecycle, bucket policies, -empty results, ARN addressing, metadata, filters, cosine distance, atomic upserts, -pagination, segmented listing, authentication, and writer loss. +signed policy grants and denies, encryption settings, empty results, ARN validation, +metadata, filters, cosine distance, atomic upserts, +catalog/vector pagination, index deletion and recreation, segmented listing, +authentication, and writer loss. Server logs and data stay in pytest's temporary directory for diagnosis. Every child process stops on exit. To select a volume server, set diff --git a/test/vectors/conftest.py b/test/vectors/conftest.py index b395f13136..dfb5ab116d 100644 --- a/test/vectors/conftest.py +++ b/test/vectors/conftest.py @@ -155,3 +155,13 @@ def vectors(cluster): return boto3.client("s3vectors", endpoint_url=cluster["endpoint"], aws_access_key_id="test", aws_secret_access_key="test", region_name="us-east-1", config=Config(connect_timeout=3, read_timeout=20, retries={"max_attempts": 0})) + + +@pytest.fixture(scope="module") +def vector_clients(vectors): + return { + name: boto3.client("s3vectors", endpoint_url=vectors.meta.endpoint_url, + aws_access_key_id=name, aws_secret_access_key=name + "-secret", + region_name="us-east-1", config=vectors.meta.config) + for name in ("reader", "writer") + } diff --git a/test/vectors/s3.json b/test/vectors/s3.json index a95df73362..436bd30be9 100644 --- a/test/vectors/s3.json +++ b/test/vectors/s3.json @@ -15,6 +15,30 @@ "Tagging", "Write" ] + }, + { + "name": "reader", + "credentials": [ + { + "accessKey": "reader", + "secretKey": "reader-secret" + } + ], + "actions": [ + "Read" + ] + }, + { + "name": "writer", + "credentials": [ + { + "accessKey": "writer", + "secretKey": "writer-secret" + } + ], + "actions": [ + "Write" + ] } ] } diff --git a/test/vectors/s3vectors_test.py b/test/vectors/s3vectors_test.py index 23f3fc3ba0..abd06d3ea4 100644 --- a/test/vectors/s3vectors_test.py +++ b/test/vectors/s3vectors_test.py @@ -85,6 +85,10 @@ def test_vector_bucket_policy(vectors): ], } ) + invalid = json.loads(policy) + invalid["Statement"][0]["Action"] = "s3tables:*" + expect_error("ValidationException", vectors.put_vector_bucket_policy, + vectorBucketName=BUCKET, policy=json.dumps(invalid)) vectors.put_vector_bucket_policy(vectorBucketName=BUCKET, policy=policy) got = vectors.get_vector_bucket_policy(vectorBucketName=BUCKET)["policy"] assert json.loads(got) == json.loads(policy) @@ -217,6 +221,141 @@ def test_delete_bucket_conflict(vectors, index): assert vectors.get_index(**index)["index"]["dimension"] == 4 +def test_vector_policy_permissions(vectors, vector_clients, index): + reader, writer = vector_clients["reader"], vector_clients["writer"] + arn = vectors.get_index(**index)["index"]["indexArn"] + row = {"key": "policy-row", "data": {"float32": [1, 0, 0, 0]}, "metadata": {"owner": "writer"}} + expect_error("AccessDeniedException", writer.put_vectors, **index, vectors=[row]) + expect_error("AccessDeniedException", reader.get_vectors, **index, keys=[row["key"]]) + statements = [ + {"Effect": "Allow", "Principal": {"AWS": "writer"}, + "Action": "s3vectors:PutVectors", "Resource": arn}, + {"Effect": "Allow", "Principal": {"AWS": "reader"}, + "Action": ["s3vectors:GetVectors", "s3vectors:QueryVectors"], "Resource": arn}, + ] + + def put_policy(): + vectors.put_vector_bucket_policy(vectorBucketName=index["vectorBucketName"], + policy=json.dumps({"Version": "2012-10-17", "Statement": statements})) + + put_policy() + writer.put_vectors(**index, vectors=[row]) + assert reader.get_vectors(**index, keys=[row["key"]], returnData=True, returnMetadata=True)["vectors"] == [row] + assert reader.query_vectors(**index, queryVector=row["data"], topK=1)["vectors"][0]["key"] == row["key"] + expect_error("AccessDeniedException", reader.put_vectors, **index, vectors=[row]) + expect_error("AccessDeniedException", writer.get_vectors, **index, keys=[row["key"]]) + expect_error("AccessDeniedException", reader.delete_index, **index) + expect_error("AccessDeniedException", reader.list_vectors, **index) + other = dict(index, indexName="other-index") + vectors.create_index(**other, dataType="float32", dimension=4, distanceMetric="euclidean") + try: + expect_error("AccessDeniedException", reader.get_vectors, **other, keys=[row["key"]]) + expect_error("AccessDeniedException", writer.put_vectors, **other, vectors=[row]) + statements.append({"Effect": "Deny", "Principal": {"AWS": "writer"}, + "Action": "s3vectors:PutVectors", "Resource": arn}) + put_policy() + expect_error("AccessDeniedException", writer.put_vectors, **index, vectors=[row]) + assert reader.get_vectors(**index, keys=[row["key"]])["vectors"][0]["key"] == row["key"] + vectors.delete_vector_bucket_policy(vectorBucketName=index["vectorBucketName"]) + expect_error("AccessDeniedException", reader.get_vectors, **index, keys=[row["key"]]) + expect_error("AccessDeniedException", writer.put_vectors, **index, vectors=[row]) + vectors.put_vectors(**index, vectors=[row]) + finally: + vectors.delete_index(**other) + + +def test_vector_bucket_encryption(vectors): + bucket = "encryption-" + uuid.uuid4().hex[:16] + config = {"sseType": "AES256"} + vectors.create_vector_bucket(vectorBucketName=bucket, encryptionConfiguration=config) + try: + assert vectors.get_vector_bucket(vectorBucketName=bucket)["vectorBucket"]["encryptionConfiguration"] == config + expect_error("ValidationException", vectors.create_vector_bucket, vectorBucketName=bucket + "-kms", + encryptionConfiguration={"sseType": "aws:kms", "kmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/test"}) + assert vectors.list_vector_buckets(prefix=bucket + "-kms")["vectorBuckets"] == [] + finally: + vectors.delete_vector_bucket(vectorBucketName=bucket) + + +def test_catalog_pagination(vectors, index): + prefix = "pages-" + uuid.uuid4().hex[:16] + buckets, indexes = [], [] + try: + for i in range(3): + bucket = f"{prefix}-{i}" + vectors.create_vector_bucket(vectorBucketName=bucket) + buckets.append(bucket) + for name in ("idx-a", "idx-b"): + args = dict(index, indexName=name) + vectors.create_index(**args, dataType="float32", dimension=4, distanceMetric="euclidean") + indexes.append(args) + pages = list(vectors.get_paginator("list_vector_buckets").paginate( + prefix=prefix, PaginationConfig={"PageSize": 1})) + assert all(len(page["vectorBuckets"]) <= 1 for page in pages) + assert sorted(b["vectorBucketName"] for page in pages for b in page["vectorBuckets"]) == buckets + pages = list(vectors.get_paginator("list_indexes").paginate( + vectorBucketName=index["vectorBucketName"], prefix="idx", PaginationConfig={"PageSize": 1})) + assert all(len(page["indexes"]) <= 1 for page in pages) + assert sorted(i["indexName"] for page in pages for i in page["indexes"]) == [INDEX, "idx-a", "idx-b"] + expect_error("ValidationException", vectors.list_vector_buckets, maxResults=501) + expect_error("ValidationException", vectors.list_indexes, vectorBucketName=index["vectorBucketName"], maxResults=501) + finally: + for args in indexes: + vectors.delete_index(**args) + for bucket in buckets: + vectors.delete_vector_bucket(vectorBucketName=bucket) + + +def test_delete_and_recreate_index(vectors, index): + vectors.put_vectors(**index, vectors=[{"key": "old", "data": {"float32": [1, 0, 0, 0]}}]) + vectors.delete_index(**index) + expect_error("NotFoundException", vectors.get_index, **index) + vectors.create_index(**index, dataType="float32", dimension=2, distanceMetric="cosine") + assert vectors.list_vectors(**index)["vectors"] == [] + row = {"key": "new", "data": {"float32": [1, 0]}} + vectors.put_vectors(**index, vectors=[row]) + assert vectors.get_vectors(**index, keys=["old", "new"], returnData=True)["vectors"] == [row] + result = vectors.query_vectors(**index, queryVector=row["data"], topK=1, returnDistance=True) + assert result["distanceMetric"] == "cosine" + assert result["vectors"][0]["key"] == "new" + assert result["vectors"][0]["distance"] == pytest.approx(0, abs=1e-6) + + +def test_index_arn_validation(vectors, index): + arn = vectors.get_index(**index)["index"]["indexArn"] + foreign = [] + for field, value in ((1, "aws-cn"), (3, "eu-west-1"), (4, "999999999999")): + parts = arn.split(":", 5) + assert parts[field] != value + parts[field] = value + foreign.append(":".join(parts)) + for invalid in ( + *foreign, + arn.replace(":s3vectors:", ":s3tables:"), + arn.replace(":s3vectors:", ":s3:"), + arn + "/index/extra", + arn + "/", + ): + expect_error("ValidationException", vectors.get_index, indexArn=invalid) + expect_error("ValidationException", vectors.delete_index, indexArn=invalid) + expect_error("ValidationException", vectors.get_vector_bucket, vectorBucketArn=arn) + expect_error("ValidationException", vectors.list_indexes, vectorBucketArn=arn) + bucket_arn = arn.rsplit("/index/", 1)[0] + for request in ( + {"vectorBucketName": "other-" + index["vectorBucketName"], "indexArn": arn}, + {"indexName": "other-" + index["indexName"], "indexArn": arn}, + ): + expect_error("ValidationException", vectors.get_index, **request) + expect_error("ValidationException", vectors.delete_index, **request) + for operation in (vectors.get_vector_bucket, vectors.list_indexes, vectors.delete_vector_bucket): + expect_error("ValidationException", operation, vectorBucketName="other-" + index["vectorBucketName"], + vectorBucketArn=bucket_arn) + assert vectors.get_index(**index, indexArn=arn)["index"]["indexArn"] == arn + assert vectors.get_vector_bucket(vectorBucketName=index["vectorBucketName"], + vectorBucketArn=bucket_arn)["vectorBucket"]["vectorBucketArn"] == bucket_arn + assert vectors.get_index(**index)["index"]["indexArn"] == arn + + def test_invalid_credentials(cluster): client = boto3.client("s3vectors", endpoint_url=cluster["endpoint"], aws_access_key_id="unknown", aws_secret_access_key="unknown", region_name="us-east-1") diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index c5ad9b6135..7a0ef0e279 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -10,6 +10,7 @@ import ( "strings" "time" + awsarn "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/google/uuid" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -58,7 +59,7 @@ func (s *Server) s3tablesArn(bucket string) string { return fmt.Sprintf("arn:aws:s3tables:%s:%s:bucket/%s", s.region, s.accountID, bucket) } -func arnParts(v *vectorBucketARNs, name, arn string) error { +func (s *Server) arnParts(v *vectorBucketARNs, name, arn string) error { if arn == "" { v.bucket = name return nil @@ -67,49 +68,52 @@ func arnParts(v *vectorBucketARNs, name, arn string) error { v.bucket = arn return nil } - i := strings.Index(arn, "bucket/") - if i < 0 { + parsed, err := awsarn.Parse(arn) + if err != nil { return fmt.Errorf("malformed ARN %q", arn) } - rest := arn[i+len("bucket/"):] - parts := strings.Split(rest, "/index/") - v.bucket = parts[0] - if v.bucket == "" { + if !strings.HasPrefix(arn, s.vectorBucketArn("")) { + return fmt.Errorf("ARN %q does not belong to this vector namespace", arn) + } + parts := strings.Split(parsed.Resource, "/") + if (len(parts) != 2 && len(parts) != 4) || parts[0] != "bucket" || parts[1] == "" { return fmt.Errorf("malformed ARN %q", arn) } - if len(parts) == 2 { - v.index = parts[1] + if len(parts) == 4 && (parts[2] != "index" || parts[3] == "") { + return fmt.Errorf("malformed ARN %q", arn) + } + v.bucket, v.index = parts[1], "" + if len(parts) == 4 { + v.index = parts[3] } return nil } -// bucketARNs resolves the bucket a bucket-level request names and rejects -// forms that point at a different resource: an index ARN names a table, and an -// ARN minted for another service names that service's bucket. -func bucketARNs(name, arn string) (vectorBucketARNs, error) { +// bucketARNs resolves bucket selectors and rejects index ARNs. +func (s *Server) bucketARNs(name, arn string) (vectorBucketARNs, error) { var v vectorBucketARNs - if err := arnParts(&v, name, arn); err != nil { + if err := s.arnParts(&v, name, arn); err != nil { return v, err } if v.index != "" { return v, fmt.Errorf("vectorBucketArn %q names an index, not a bucket", arn) } - if strings.HasPrefix(arn, "arn:") { - if p := strings.SplitN(arn, ":", 6); len(p) > 2 && p[2] != "s3vectors" { - return v, fmt.Errorf("vectorBucketArn %q is not an s3vectors ARN", arn) - } + if name != "" && name != v.bucket { + return v, fmt.Errorf("vectorBucketName and vectorBucketArn name different buckets") } return v, nil } -// resolveIndexARNs fills bucket and index from any combination of plain names, -// a bucket ARN, and a full index ARN — callers may supply only an indexArn. -func resolveIndexARNs(bucketName, bucketArn, indexName, indexArn string) (vectorBucketARNs, error) { +// resolveIndexARNs resolves names, a bucket ARN, or an index ARN. +func (s *Server) resolveIndexARNs(bucketName, bucketArn, indexName, indexArn string) (vectorBucketARNs, error) { var v vectorBucketARNs if strings.HasPrefix(indexArn, "arn:") { - if err := arnParts(&v, "", indexArn); err != nil { + if err := s.arnParts(&v, "", indexArn); err != nil { return v, err } + if v.index == "" { + return v, fmt.Errorf("indexArn %q does not name an index", indexArn) + } } else if indexArn != "" { return v, fmt.Errorf("indexArn must be an ARN, got %q", indexArn) } @@ -117,15 +121,24 @@ func resolveIndexARNs(bucketName, bucketArn, indexName, indexArn string) (vector if !strings.HasPrefix(bucketArn, "arn:") { return v, fmt.Errorf("vectorBucketArn must be an ARN, got %q", bucketArn) } - if err := arnParts(&v, "", bucketArn); err != nil { + bucket, err := s.bucketARNs("", bucketArn) + if err != nil { return v, err } + if v.bucket != "" && v.bucket != bucket.bucket { + return v, fmt.Errorf("vectorBucketArn and indexArn name different buckets") + } + v.bucket = bucket.bucket } if v.bucket == "" { v.bucket = bucketName + } else if bucketName != "" && bucketName != v.bucket { + return v, fmt.Errorf("vectorBucketName and ARN bucket name different buckets") } if v.index == "" { v.index = indexName + } else if indexName != "" && indexName != v.index { + return v, fmt.Errorf("indexName and indexArn name different indexes") } if v.bucket == "" || v.index == "" { return v, fmt.Errorf("vectorBucketName and indexName (or indexArn) are required") @@ -272,7 +285,7 @@ func (s *Server) handleGetVectorBucket(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - arns, err := bucketARNs(req.VectorBucketName, req.VectorBucketArn) + arns, err := s.bucketARNs(req.VectorBucketName, req.VectorBucketArn) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -365,7 +378,7 @@ func (s *Server) handleDeleteVectorBucket(w http.ResponseWriter, r *http.Request writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - arns, err := bucketARNs(req.VectorBucketName, req.VectorBucketArn) + arns, err := s.bucketARNs(req.VectorBucketName, req.VectorBucketArn) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -763,7 +776,7 @@ func (s *Server) handlePutVectorBucketPolicy(w http.ResponseWriter, r *http.Requ writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - arns, err := bucketARNs(req.VectorBucketName, req.VectorBucketArn) + arns, err := s.bucketARNs(req.VectorBucketName, req.VectorBucketArn) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -811,7 +824,7 @@ func (s *Server) handleGetVectorBucketPolicy(w http.ResponseWriter, r *http.Requ writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - arns, err := bucketARNs(req.VectorBucketName, req.VectorBucketArn) + arns, err := s.bucketARNs(req.VectorBucketName, req.VectorBucketArn) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -838,7 +851,7 @@ func (s *Server) handleDeleteVectorBucketPolicy(w http.ResponseWriter, r *http.R writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - arns, err := bucketARNs(req.VectorBucketName, req.VectorBucketArn) + arns, err := s.bucketARNs(req.VectorBucketName, req.VectorBucketArn) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -903,7 +916,7 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", "unknown distanceMetric "+req.DistanceMetric) return } - arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) + arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -1060,7 +1073,7 @@ func (s *Server) handleGetIndex(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) + arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -1088,7 +1101,7 @@ func (s *Server) handleListIndexes(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - arns, err := bucketARNs(req.VectorBucketName, req.VectorBucketArn) + arns, err := s.bucketARNs(req.VectorBucketName, req.VectorBucketArn) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -1149,7 +1162,7 @@ func (s *Server) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) + arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -1210,7 +1223,7 @@ func (s *Server) handlePutVectors(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) + arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -1294,7 +1307,7 @@ func (s *Server) handleGetVectors(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) + arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -1328,7 +1341,7 @@ func (s *Server) handleListVectors(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) + arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -1456,7 +1469,7 @@ func (s *Server) handleDeleteVectors(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) + arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return @@ -1755,7 +1768,7 @@ func (s *Server) handleQueryVectors(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) + arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn) if err != nil { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return diff --git a/weed/s3api/s3vectors/handlers_test.go b/weed/s3api/s3vectors/handlers_test.go index e50efa8f9c..045e51ae51 100644 --- a/weed/s3api/s3vectors/handlers_test.go +++ b/weed/s3api/s3vectors/handlers_test.go @@ -230,35 +230,46 @@ func TestListRequestMaxResultsBounds(t *testing.T) { } func TestArnParts(t *testing.T) { + s := &Server{region: "r", accountID: "a"} var v vectorBucketARNs - if err := arnParts(&v, "plain", ""); err != nil || v.bucket != "plain" { + if err := s.arnParts(&v, "plain", ""); err != nil || v.bucket != "plain" { t.Fatalf("plain name: bucket=%q err=%v", v.bucket, err) } - if err := arnParts(&v, "", "arn:aws:s3vectors:r:a:bucket/mybucket"); err != nil || v.bucket != "mybucket" || v.index != "" { + if err := s.arnParts(&v, "", "arn:aws:s3vectors:r:a:bucket/mybucket"); err != nil || v.bucket != "mybucket" || v.index != "" { t.Fatalf("bucket ARN: %+v err=%v", v, err) } v = vectorBucketARNs{} - if err := arnParts(&v, "", "arn:aws:s3vectors:r:a:bucket/mybucket/index/myindex"); err != nil || v.bucket != "mybucket" || v.index != "myindex" { + if err := s.arnParts(&v, "", "arn:aws:s3vectors:r:a:bucket/mybucket/index/myindex"); err != nil || v.bucket != "mybucket" || v.index != "myindex" { t.Fatalf("index ARN: %+v err=%v", v, err) } for _, bad := range []string{ "arn:aws:s3vectors:r:a:bucket/", "arn:aws:s3vectors:r:a:bucket//index/i", "arn:aws:s3vectors:r:a:bucket", + "arn:aws:s3tables:r:a:bucket/b/index/i", + "arn:aws:s3:r:a:bucket/b/index/i", + "arn:aws:s3vectors:r:a:prefix/bucket/b/index/i", + "arn:aws:s3vectors:r:a:bucket/b/index/", + "arn:aws:s3vectors:r:a:bucket/b/index/i/index/j", + "arn:aws:s3vectors:r:a:bucket/b/index/i/", + "arn:aws-cn:s3vectors:r:a:bucket/b/index/i", + "arn:aws:s3vectors:other:a:bucket/b/index/i", + "arn:aws:s3vectors:r:other:bucket/b/index/i", } { v = vectorBucketARNs{} - if err := arnParts(&v, "", bad); err == nil { + if err := s.arnParts(&v, "", bad); err == nil { t.Fatalf("ARN %q should be rejected", bad) } } } func TestBucketARNs(t *testing.T) { - if v, err := bucketARNs("plain", ""); err != nil || v.bucket != "plain" { + s := &Server{region: "r", accountID: "a"} + if v, err := s.bucketARNs("plain", ""); err != nil || v.bucket != "plain" { t.Fatalf("plain name: %+v err=%v", v, err) } - if v, err := bucketARNs("", "arn:aws:s3vectors:r:a:bucket/mybucket"); err != nil || v.bucket != "mybucket" { + if v, err := s.bucketARNs("", "arn:aws:s3vectors:r:a:bucket/mybucket"); err != nil || v.bucket != "mybucket" { t.Fatalf("bucket ARN: %+v err=%v", v, err) } for _, bad := range []string{ @@ -266,28 +277,54 @@ func TestBucketARNs(t *testing.T) { "arn:aws:s3tables:r:a:bucket/b", "arn:aws:s3:r:a:bucket/b", } { - if _, err := bucketARNs("", bad); err == nil { + if _, err := s.bucketARNs("", bad); err == nil { t.Fatalf("ARN %q should be rejected for a bucket operation", bad) } } + if _, err := s.bucketARNs("a", "arn:aws:s3vectors:r:a:bucket/b"); err == nil { + t.Fatal("conflicting vectorBucketName and vectorBucketArn should be rejected") + } } func TestResolveIndexARNs(t *testing.T) { - arns, err := resolveIndexARNs("", "", "", "arn:aws:s3vectors:r:a:bucket/b/index/i") + s := &Server{region: "r", accountID: "a"} + arns, err := s.resolveIndexARNs("", "", "", "arn:aws:s3vectors:r:a:bucket/b/index/i") if err != nil || arns.bucket != "b" || arns.index != "i" { t.Fatalf("indexArn only: %+v err=%v", arns, err) } - arns, err = resolveIndexARNs("b", "", "i", "") + arns, err = s.resolveIndexARNs("b", "", "i", "") if err != nil || arns.bucket != "b" || arns.index != "i" { t.Fatalf("names only: %+v err=%v", arns, err) } - if _, err := resolveIndexARNs("", "not-an-arn", "i", ""); err == nil { + arns, err = s.resolveIndexARNs("", "arn:aws:s3vectors:r:a:bucket/b", "", "arn:aws:s3vectors:r:a:bucket/b/index/i") + if err != nil || arns.bucket != "b" || arns.index != "i" { + t.Fatalf("bucket and index ARNs: %+v err=%v", arns, err) + } + for _, tc := range []struct{ bucketARN, indexARN, indexName string }{ + {"", "arn:aws:s3vectors:r:a:bucket/b", "i"}, + {"arn:aws:s3vectors:r:a:bucket/b/index/i", "", "i"}, + {"arn:aws:s3vectors:r:a:bucket/other", "arn:aws:s3vectors:r:a:bucket/b/index/i", ""}, + {"arn:aws-cn:s3vectors:r:a:bucket/b", "arn:aws:s3vectors:r:a:bucket/b/index/i", ""}, + {"arn:aws:s3vectors:other:a:bucket/b", "arn:aws:s3vectors:r:a:bucket/b/index/i", ""}, + {"arn:aws:s3vectors:r:other:bucket/b", "arn:aws:s3vectors:r:a:bucket/b/index/i", ""}, + } { + if _, err := s.resolveIndexARNs("", tc.bucketARN, tc.indexName, tc.indexARN); err == nil { + t.Errorf("accepted incompatible ARNs: %+v", tc) + } + } + if _, err := s.resolveIndexARNs("other", "", "", "arn:aws:s3vectors:r:a:bucket/b/index/i"); err == nil { + t.Fatal("conflicting vectorBucketName should be rejected") + } + if _, err := s.resolveIndexARNs("", "", "other", "arn:aws:s3vectors:r:a:bucket/b/index/i"); err == nil { + t.Fatal("conflicting indexName should be rejected") + } + if _, err := s.resolveIndexARNs("", "not-an-arn", "i", ""); err == nil { t.Fatal("non-ARN vectorBucketArn should be rejected") } - if _, err := resolveIndexARNs("", "", "", ""); err == nil { + if _, err := s.resolveIndexARNs("", "", "", ""); err == nil { t.Fatal("missing identifiers should be rejected") } - if _, err := resolveIndexARNs("b", "", "", ""); err == nil { + if _, err := s.resolveIndexARNs("b", "", "", ""); err == nil { t.Fatal("missing indexName should be rejected") } }