Github-Mirror/seaweedfs
1
0 Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-11-06 03:57:48 +01:00
Code Issues Packages Projects Releases Wiki Activity

s3vectors: validate index ARNs and expand live integration coverage (#12222)

Browse Source 
* test: align vector bucket policy coverage with vector actions

* s3vectors: validate ARN service and resource shape before resolving indexes

* test: exercise signed vector bucket policies and catalog lifecycle

* ci: require the expanded vector integration matrix and all gateway tests

* test: reject index ARNs in bucket-level API requests

* s3vectors: scope ARN resolution to the gateway namespace

* s3vectors: reject name fields that conflict with resolved ARNs

bucketARNs and resolveIndexARNs already reject an indexArn that names a
different bucket than vectorBucketArn, but a vectorBucketName or
indexName disagreeing with the ARNs was silently dropped. Validate the
plain names against the resolved ARN values the same way.

* test: cover conflicting vector names and ARNs in live requests

---------

Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Chris LuandDevin authored and GitHub committed 2026-10-29 17:57:11 -07:00
1 parent cc5912b8b8
commit d0441bbdbb
7 files changed
+282 -55
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

weed/s3api/s3vectors/handlers.go
+50 -37
View File
@@ -10,6 +10,7 @@ import (
"strings"
"time"
awsarn "github.com/aws/aws-sdk-go-v2/aws/arn"
"github.com/google/uuid"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -58,7 +59,7 @@ func (s *Server) s3tablesArn(bucket string) string {
return fmt.Sprintf("arn:aws:s3tables:%s:%s:bucket/%s", s.region, s.accountID, bucket)
}
func arnParts(v *vectorBucketARNs, name, arn string) error {
func (s *Server) arnParts(v *vectorBucketARNs, name, arn string) error {
if arn == "" {
v.bucket = name
return nil
@@ -67,49 +68,52 @@ func arnParts(v *vectorBucketARNs, name, arn string) error {
v.bucket = arn
return nil
}
i := strings.Index(arn, "bucket/")
if i < 0 {
parsed, err := awsarn.Parse(arn)
if err != nil {
return fmt.Errorf("malformed ARN %q", arn)
}
rest := arn[i+len("bucket/"):]
parts := strings.Split(rest, "/index/")
v.bucket = parts[0]
if v.bucket == "" {
if !strings.HasPrefix(arn, s.vectorBucketArn("")) {
return fmt.Errorf("ARN %q does not belong to this vector namespace", arn)
}
parts := strings.Split(parsed.Resource, "/")
if (len(parts) != 2 && len(parts) != 4) || parts[0] != "bucket" || parts[1] == "" {
return fmt.Errorf("malformed ARN %q", arn)
}
if len(parts) == 2 {
v.index = parts[1]
if len(parts) == 4 && (parts[2] != "index" || parts[3] == "") {
return fmt.Errorf("malformed ARN %q", arn)
}
v.bucket, v.index = parts[1], ""
if len(parts) == 4 {
v.index = parts[3]
}
return nil
}
// bucketARNs resolves the bucket a bucket-level request names and rejects
// forms that point at a different resource: an index ARN names a table, and an
// ARN minted for another service names that service's bucket.
func bucketARNs(name, arn string) (vectorBucketARNs, error) {
// bucketARNs resolves bucket selectors and rejects index ARNs.
func (s *Server) bucketARNs(name, arn string) (vectorBucketARNs, error) {
var v vectorBucketARNs
if err := arnParts(&v, name, arn); err != nil {
if err := s.arnParts(&v, name, arn); err != nil {
return v, err
}
if v.index != "" {
return v, fmt.Errorf("vectorBucketArn %q names an index, not a bucket", arn)
}
if strings.HasPrefix(arn, "arn:") {
if p := strings.SplitN(arn, ":", 6); len(p) > 2 && p[2] != "s3vectors" {
return v, fmt.Errorf("vectorBucketArn %q is not an s3vectors ARN", arn)
}
if name != "" && name != v.bucket {
return v, fmt.Errorf("vectorBucketName and vectorBucketArn name different buckets")
}
return v, nil
}
// resolveIndexARNs fills bucket and index from any combination of plain names,
// a bucket ARN, and a full index ARN — callers may supply only an indexArn.
func resolveIndexARNs(bucketName, bucketArn, indexName, indexArn string) (vectorBucketARNs, error) {
// resolveIndexARNs resolves names, a bucket ARN, or an index ARN.
func (s *Server) resolveIndexARNs(bucketName, bucketArn, indexName, indexArn string) (vectorBucketARNs, error) {
var v vectorBucketARNs
if strings.HasPrefix(indexArn, "arn:") {
if err := arnParts(&v, "", indexArn); err != nil {
if err := s.arnParts(&v, "", indexArn); err != nil {
return v, err
}
if v.index == "" {
return v, fmt.Errorf("indexArn %q does not name an index", indexArn)
}
} else if indexArn != "" {
return v, fmt.Errorf("indexArn must be an ARN, got %q", indexArn)
}
@@ -117,15 +121,24 @@ func resolveIndexARNs(bucketName, bucketArn, indexName, indexArn string) (vector
if !strings.HasPrefix(bucketArn, "arn:") {
return v, fmt.Errorf("vectorBucketArn must be an ARN, got %q", bucketArn)
}
if err := arnParts(&v, "", bucketArn); err != nil {
bucket, err := s.bucketARNs("", bucketArn)
if err != nil {
return v, err
}
if v.bucket != "" && v.bucket != bucket.bucket {
return v, fmt.Errorf("vectorBucketArn and indexArn name different buckets")
}
v.bucket = bucket.bucket
}
if v.bucket == "" {
v.bucket = bucketName
} else if bucketName != "" && bucketName != v.bucket {
return v, fmt.Errorf("vectorBucketName and ARN bucket name different buckets")
}
if v.index == "" {
v.index = indexName
} else if indexName != "" && indexName != v.index {
return v, fmt.Errorf("indexName and indexArn name different indexes")
}
if v.bucket == "" || v.index == "" {
return v, fmt.Errorf("vectorBucketName and indexName (or indexArn) are required")
@@ -272,7 +285,7 @@ func (s *Server) handleGetVectorBucket(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
arns, err := bucketARNs(req.VectorBucketName, req.VectorBucketArn)
arns, err := s.bucketARNs(req.VectorBucketName, req.VectorBucketArn)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -365,7 +378,7 @@ func (s *Server) handleDeleteVectorBucket(w http.ResponseWriter, r *http.Request
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
arns, err := bucketARNs(req.VectorBucketName, req.VectorBucketArn)
arns, err := s.bucketARNs(req.VectorBucketName, req.VectorBucketArn)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -763,7 +776,7 @@ func (s *Server) handlePutVectorBucketPolicy(w http.ResponseWriter, r *http.Requ
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
arns, err := bucketARNs(req.VectorBucketName, req.VectorBucketArn)
arns, err := s.bucketARNs(req.VectorBucketName, req.VectorBucketArn)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -811,7 +824,7 @@ func (s *Server) handleGetVectorBucketPolicy(w http.ResponseWriter, r *http.Requ
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
arns, err := bucketARNs(req.VectorBucketName, req.VectorBucketArn)
arns, err := s.bucketARNs(req.VectorBucketName, req.VectorBucketArn)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -838,7 +851,7 @@ func (s *Server) handleDeleteVectorBucketPolicy(w http.ResponseWriter, r *http.R
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
arns, err := bucketARNs(req.VectorBucketName, req.VectorBucketArn)
arns, err := s.bucketARNs(req.VectorBucketName, req.VectorBucketArn)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -903,7 +916,7 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", "unknown distanceMetric "+req.DistanceMetric)
return
}
arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -1060,7 +1073,7 @@ func (s *Server) handleGetIndex(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -1088,7 +1101,7 @@ func (s *Server) handleListIndexes(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
arns, err := bucketARNs(req.VectorBucketName, req.VectorBucketArn)
arns, err := s.bucketARNs(req.VectorBucketName, req.VectorBucketArn)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -1149,7 +1162,7 @@ func (s *Server) handleDeleteIndex(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -1210,7 +1223,7 @@ func (s *Server) handlePutVectors(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -1294,7 +1307,7 @@ func (s *Server) handleGetVectors(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -1328,7 +1341,7 @@ func (s *Server) handleListVectors(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -1456,7 +1469,7 @@ func (s *Server) handleDeleteVectors(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
@@ -1755,7 +1768,7 @@ func (s *Server) handleQueryVectors(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
arns, err := resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
arns, err := s.resolveIndexARNs(req.VectorBucketName, req.VectorBucketArn, req.IndexName, req.IndexArn)
if err != nil {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
weed/s3api/s3vectors/handlers_test.go
+49 -12
View File
@@ -230,35 +230,46 @@ func TestListRequestMaxResultsBounds(t *testing.T) {
}
func TestArnParts(t *testing.T) {
s := &Server{region: "r", accountID: "a"}
var v vectorBucketARNs
if err := arnParts(&v, "plain", ""); err != nil || v.bucket != "plain" {
if err := s.arnParts(&v, "plain", ""); err != nil || v.bucket != "plain" {
t.Fatalf("plain name: bucket=%q err=%v", v.bucket, err)
}
if err := arnParts(&v, "", "arn:aws:s3vectors:r:a:bucket/mybucket"); err != nil || v.bucket != "mybucket" || v.index != "" {
if err := s.arnParts(&v, "", "arn:aws:s3vectors:r:a:bucket/mybucket"); err != nil || v.bucket != "mybucket" || v.index != "" {
t.Fatalf("bucket ARN: %+v err=%v", v, err)
}
v = vectorBucketARNs{}
if err := arnParts(&v, "", "arn:aws:s3vectors:r:a:bucket/mybucket/index/myindex"); err != nil || v.bucket != "mybucket" || v.index != "myindex" {
if err := s.arnParts(&v, "", "arn:aws:s3vectors:r:a:bucket/mybucket/index/myindex"); err != nil || v.bucket != "mybucket" || v.index != "myindex" {
t.Fatalf("index ARN: %+v err=%v", v, err)
}
for _, bad := range []string{
"arn:aws:s3vectors:r:a:bucket/",
"arn:aws:s3vectors:r:a:bucket//index/i",
"arn:aws:s3vectors:r:a:bucket",
"arn:aws:s3tables:r:a:bucket/b/index/i",
"arn:aws:s3:r:a:bucket/b/index/i",
"arn:aws:s3vectors:r:a:prefix/bucket/b/index/i",
"arn:aws:s3vectors:r:a:bucket/b/index/",
"arn:aws:s3vectors:r:a:bucket/b/index/i/index/j",
"arn:aws:s3vectors:r:a:bucket/b/index/i/",
"arn:aws-cn:s3vectors:r:a:bucket/b/index/i",
"arn:aws:s3vectors:other:a:bucket/b/index/i",
"arn:aws:s3vectors:r:other:bucket/b/index/i",
} {
v = vectorBucketARNs{}
if err := arnParts(&v, "", bad); err == nil {
if err := s.arnParts(&v, "", bad); err == nil {
t.Fatalf("ARN %q should be rejected", bad)
}
}
}
func TestBucketARNs(t *testing.T) {
if v, err := bucketARNs("plain", ""); err != nil || v.bucket != "plain" {
s := &Server{region: "r", accountID: "a"}
if v, err := s.bucketARNs("plain", ""); err != nil || v.bucket != "plain" {
t.Fatalf("plain name: %+v err=%v", v, err)
}
if v, err := bucketARNs("", "arn:aws:s3vectors:r:a:bucket/mybucket"); err != nil || v.bucket != "mybucket" {
if v, err := s.bucketARNs("", "arn:aws:s3vectors:r:a:bucket/mybucket"); err != nil || v.bucket != "mybucket" {
t.Fatalf("bucket ARN: %+v err=%v", v, err)
}
for _, bad := range []string{
@@ -266,28 +277,54 @@ func TestBucketARNs(t *testing.T) {
"arn:aws:s3tables:r:a:bucket/b",
"arn:aws:s3:r:a:bucket/b",
} {
if _, err := bucketARNs("", bad); err == nil {
if _, err := s.bucketARNs("", bad); err == nil {
t.Fatalf("ARN %q should be rejected for a bucket operation", bad)
}
}
if _, err := s.bucketARNs("a", "arn:aws:s3vectors:r:a:bucket/b"); err == nil {
t.Fatal("conflicting vectorBucketName and vectorBucketArn should be rejected")
}
}
func TestResolveIndexARNs(t *testing.T) {
arns, err := resolveIndexARNs("", "", "", "arn:aws:s3vectors:r:a:bucket/b/index/i")
s := &Server{region: "r", accountID: "a"}
arns, err := s.resolveIndexARNs("", "", "", "arn:aws:s3vectors:r:a:bucket/b/index/i")
if err != nil || arns.bucket != "b" || arns.index != "i" {
t.Fatalf("indexArn only: %+v err=%v", arns, err)
}
arns, err = resolveIndexARNs("b", "", "i", "")
arns, err = s.resolveIndexARNs("b", "", "i", "")
if err != nil || arns.bucket != "b" || arns.index != "i" {
t.Fatalf("names only: %+v err=%v", arns, err)
}
if _, err := resolveIndexARNs("", "not-an-arn", "i", ""); err == nil {
arns, err = s.resolveIndexARNs("", "arn:aws:s3vectors:r:a:bucket/b", "", "arn:aws:s3vectors:r:a:bucket/b/index/i")
if err != nil || arns.bucket != "b" || arns.index != "i" {
t.Fatalf("bucket and index ARNs: %+v err=%v", arns, err)
}
for _, tc := range []struct{ bucketARN, indexARN, indexName string }{
{"", "arn:aws:s3vectors:r:a:bucket/b", "i"},
{"arn:aws:s3vectors:r:a:bucket/b/index/i", "", "i"},
{"arn:aws:s3vectors:r:a:bucket/other", "arn:aws:s3vectors:r:a:bucket/b/index/i", ""},
{"arn:aws-cn:s3vectors:r:a:bucket/b", "arn:aws:s3vectors:r:a:bucket/b/index/i", ""},
{"arn:aws:s3vectors:other:a:bucket/b", "arn:aws:s3vectors:r:a:bucket/b/index/i", ""},
{"arn:aws:s3vectors:r:other:bucket/b", "arn:aws:s3vectors:r:a:bucket/b/index/i", ""},
} {
if _, err := s.resolveIndexARNs("", tc.bucketARN, tc.indexName, tc.indexARN); err == nil {
t.Errorf("accepted incompatible ARNs: %+v", tc)
}
}
if _, err := s.resolveIndexARNs("other", "", "", "arn:aws:s3vectors:r:a:bucket/b/index/i"); err == nil {
t.Fatal("conflicting vectorBucketName should be rejected")
}
if _, err := s.resolveIndexARNs("", "", "other", "arn:aws:s3vectors:r:a:bucket/b/index/i"); err == nil {
t.Fatal("conflicting indexName should be rejected")
}
if _, err := s.resolveIndexARNs("", "not-an-arn", "i", ""); err == nil {
t.Fatal("non-ARN vectorBucketArn should be rejected")
}
if _, err := resolveIndexARNs("", "", "", ""); err == nil {
if _, err := s.resolveIndexARNs("", "", "", ""); err == nil {
t.Fatal("missing identifiers should be rejected")
}
if _, err := resolveIndexARNs("b", "", "", ""); err == nil {
if _, err := s.resolveIndexARNs("b", "", "", ""); err == nil {
t.Fatal("missing indexName should be rejected")
}
}