mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-30 08:50:48 +01:00
s3vectors: accept and return encryptionConfiguration on vector buckets (#12203)
* s3tables: carry encryptionConfiguration in table bucket metadata * s3vectors: accept and return encryptionConfiguration on vector buckets The configuration is validated (sseType AES256 or aws:kms) and stored on the bucket record, so the API round-trips what the caller declares. * s3vectors: reject caller-chosen encryption keys on vector buckets * s3tables: reject caller-chosen encryption keys on table buckets * s3tables: reject unknown sseType in bucket encryptionConfiguration The bucket check rejected aws:kms and kmsKeyArn but stored any other sseType — a made-up or mistyped value would round-trip through GetTableBucket as an encryption claim the storage does not keep. The validator now admits only the AES256 declaration, shared so the vector layer can drop its copy. * s3vectors: delegate encryption validation to the shared catalog check * s3tables: accept the sseAlgorithm name in bucket encryptionConfiguration S3 Tables declares the algorithm as sseAlgorithm while S3 Vectors uses sseType; the shared check now accepts either spelling, still admits only AES256, and still refuses any caller-chosen key. * s3vectors: call the shared encryption validator directly * s3vectors: require each API's own encryption field name The shared validator accepted sseAlgorithm on the vector path, so a CreateVectorBucket carrying the s3tables field stored a configuration GetVectorBucket could never echo back in the vectors shape (and vice versa). ValidateEncryptionConfiguration now takes the wire name its caller expects; the vectors create path normalizes to sseAlgorithm for the catalog and GetVectorBucket renames it back to sseType.
This commit is contained in:
1 parent
fc9be77dbb
commit
c7d77eb9a8
8 files changed
+140
-27
No files matched your search
@@ -39,6 +39,11 @@ func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http
|
||||
bucketFormat = normalized
|
||||
}
|
||||
|
||||
if err := ValidateEncryptionConfiguration(req.EncryptionConfiguration, "sseAlgorithm"); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
identityPolicyNames := getIdentityPolicyNames(r)
|
||||
@@ -127,10 +132,11 @@ func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http
|
||||
// Create the bucket directory and set metadata as extended attributes
|
||||
now := time.Now()
|
||||
metadata := &tableBucketMetadata{
|
||||
Name: req.Name,
|
||||
CreatedAt: now,
|
||||
OwnerAccountID: principal,
|
||||
Format: bucketFormat,
|
||||
Name: req.Name,
|
||||
CreatedAt: now,
|
||||
OwnerAccountID: principal,
|
||||
Format: bucketFormat,
|
||||
EncryptionConfiguration: req.EncryptionConfiguration,
|
||||
}
|
||||
|
||||
metadataBytes, err := json.Marshal(metadata)
|
||||
|
||||
@@ -81,11 +81,12 @@ func (h *S3TablesHandler) handleGetTableBucket(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
|
||||
resp := &GetTableBucketResponse{
|
||||
ARN: h.generateTableBucketARN(metadata.OwnerAccountID, bucketName),
|
||||
Name: metadata.Name,
|
||||
OwnerAccountID: metadata.OwnerAccountID,
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
Format: metadata.Format,
|
||||
ARN: h.generateTableBucketARN(metadata.OwnerAccountID, bucketName),
|
||||
Name: metadata.Name,
|
||||
OwnerAccountID: metadata.OwnerAccountID,
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
Format: metadata.Format,
|
||||
EncryptionConfiguration: metadata.EncryptionConfiguration,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
|
||||
@@ -97,3 +97,40 @@ func TestManagerCreateTableBucketAuthorization(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The bucket stores the declared configuration, but nothing downstream reads a
|
||||
// caller-chosen key — accepting one records a promise the data never gets.
|
||||
func TestManagerCreateTableBucketRejectsCustomerKeys(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cfg map[string]interface{}
|
||||
reaches bool
|
||||
}{
|
||||
{"no configuration", nil, true},
|
||||
{"tables default sse", map[string]interface{}{"sseAlgorithm": "AES256"}, true},
|
||||
{"vector wire name", map[string]interface{}{"sseType": "AES256"}, false},
|
||||
{"unknown sse", map[string]interface{}{"sseType": "aws:fsx"}, false},
|
||||
{"tables kms", map[string]interface{}{"sseAlgorithm": "aws:kms"}, false},
|
||||
{"conflicting types", map[string]interface{}{"sseAlgorithm": "AES256", "sseType": "aws:kms"}, false},
|
||||
{"missing algorithm", map[string]interface{}{"other": "x"}, false},
|
||||
{"customer key", map[string]interface{}{"sseType": "aws:kms", "kmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/x"}, false},
|
||||
{"key without a type", map[string]interface{}{"kmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/x"}, false},
|
||||
{"key on default sse", map[string]interface{}{"sseType": "AES256", "kmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/x"}, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
m := NewManager()
|
||||
fc := &recordingFilerClient{}
|
||||
err := m.Execute(context.Background(), fc, "CreateTableBucket", &CreateTableBucketRequest{
|
||||
Name: "testbucket",
|
||||
EncryptionConfiguration: tc.cfg,
|
||||
}, nil, s3_constants.AccountAdminId)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, tc.reaches, fc.called)
|
||||
if !tc.reaches {
|
||||
var s3Err *S3TablesError
|
||||
require.ErrorAs(t, err, &s3Err)
|
||||
assert.Equal(t, ErrCodeInvalidRequest, s3Err.Type)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -17,8 +17,9 @@ type TableBucket struct {
|
||||
}
|
||||
|
||||
type CreateTableBucketRequest struct {
|
||||
Name string `json:"name"`
|
||||
Tags map[string]string `json:"tags,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Tags map[string]string `json:"tags,omitempty"`
|
||||
EncryptionConfiguration map[string]interface{} `json:"encryptionConfiguration,omitempty"`
|
||||
// Format is the table format this bucket holds. A bucket is a catalog and a
|
||||
// catalog serves one protocol, so declaring it here is what lets a caller be
|
||||
// told where to connect. Empty means ICEBERG, which is what AWS S3 Tables
|
||||
@@ -41,7 +42,8 @@ type GetTableBucketResponse struct {
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
// Format is empty for a bucket created before formats were declared. Such a
|
||||
// bucket accepts any format, which is what it did when it was made.
|
||||
Format string `json:"format,omitempty"`
|
||||
Format string `json:"format,omitempty"`
|
||||
EncryptionConfiguration map[string]interface{} `json:"encryptionConfiguration,omitempty"`
|
||||
}
|
||||
|
||||
type ListTableBucketsRequest struct {
|
||||
|
||||
@@ -164,6 +164,34 @@ func ValidateMetadataLocation(metadataLocation, bucketName, namespace string) er
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateEncryptionConfiguration admits only the AES256 declaration, under the
|
||||
// caller's wire name — s3tables sends sseAlgorithm, s3vectors sends sseType. A
|
||||
// caller-chosen key is never read back by the storage path, so recording one
|
||||
// would claim protection the data does not get.
|
||||
func ValidateEncryptionConfiguration(cfg map[string]interface{}, sseField string) error {
|
||||
if len(cfg) == 0 {
|
||||
return nil
|
||||
}
|
||||
otherField := "sseAlgorithm"
|
||||
if sseField == otherField {
|
||||
otherField = "sseType"
|
||||
}
|
||||
if _, ok := cfg[otherField]; ok {
|
||||
return fmt.Errorf("encryptionConfiguration field is %s", sseField)
|
||||
}
|
||||
value, ok := cfg[sseField].(string)
|
||||
if !ok {
|
||||
return fmt.Errorf("encryptionConfiguration requires %s", sseField)
|
||||
}
|
||||
if value != "AES256" {
|
||||
return fmt.Errorf("unsupported %s %q", sseField, value)
|
||||
}
|
||||
if _, ok := cfg["kmsKeyArn"]; ok {
|
||||
return fmt.Errorf("kmsKeyArn is not supported")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseS3Location(location string) (bucket, tablePath string, err error) {
|
||||
if !strings.HasPrefix(location, "s3://") {
|
||||
return "", "", fmt.Errorf("unsupported location: %s", location)
|
||||
@@ -197,10 +225,11 @@ func GetTableObjectBucketPath(bucketName string) string {
|
||||
// Metadata structures
|
||||
|
||||
type tableBucketMetadata struct {
|
||||
Name string `json:"name"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
OwnerAccountID string `json:"ownerAccountId"`
|
||||
Format string `json:"format,omitempty"`
|
||||
Name string `json:"name"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
OwnerAccountID string `json:"ownerAccountId"`
|
||||
Format string `json:"format,omitempty"`
|
||||
EncryptionConfiguration map[string]interface{} `json:"encryptionConfiguration,omitempty"`
|
||||
}
|
||||
|
||||
// namespaceMetadata stores metadata for a namespace
|
||||
|
||||
@@ -275,16 +275,22 @@ func (s *Server) handleCreateVectorBucket(w http.ResponseWriter, r *http.Request
|
||||
writeError(w, http.StatusBadRequest, "ValidationException", "vectorBucketName is required")
|
||||
return
|
||||
}
|
||||
// The catalog has no encryption boundary to honor the configuration
|
||||
// against; accepting it would silently promise semantics we do not keep.
|
||||
if len(req.EncryptionConfiguration) > 0 {
|
||||
writeError(w, http.StatusBadRequest, "ValidationException",
|
||||
"encryptionConfiguration is not supported")
|
||||
if err := s3tables.ValidateEncryptionConfiguration(req.EncryptionConfiguration, "sseType"); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
|
||||
return
|
||||
}
|
||||
// The catalog keeps the s3tables field name; GetVectorBucket renames it back.
|
||||
enc := req.EncryptionConfiguration
|
||||
if len(enc) != 0 {
|
||||
enc = map[string]interface{}{"sseAlgorithm": enc["sseType"]}
|
||||
}
|
||||
var resp map[string]interface{}
|
||||
err := s.catalog(r.Context(), "CreateTableBucket",
|
||||
&s3tables.CreateTableBucketRequest{Name: req.VectorBucketName, Format: s3tables.FormatLance}, &resp, r)
|
||||
&s3tables.CreateTableBucketRequest{
|
||||
Name: req.VectorBucketName,
|
||||
Format: s3tables.FormatLance,
|
||||
EncryptionConfiguration: enc,
|
||||
}, &resp, r)
|
||||
if err != nil {
|
||||
catalogErr(w, err)
|
||||
return
|
||||
@@ -317,11 +323,18 @@ func (s *Server) handleGetVectorBucket(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
created, _ := time.Parse(time.RFC3339, fmt.Sprint(resp["createdAt"]))
|
||||
writeJSON(w, http.StatusOK, &GetVectorBucketResponse{VectorBucket: &VectorBucket{
|
||||
bucket := &VectorBucket{
|
||||
VectorBucketArn: s.vectorBucketArn(arns.bucket),
|
||||
VectorBucketName: arns.bucket,
|
||||
CreationTime: created,
|
||||
}})
|
||||
}
|
||||
if cfg, ok := resp["encryptionConfiguration"].(map[string]interface{}); ok {
|
||||
if v, ok := cfg["sseAlgorithm"]; ok {
|
||||
cfg = map[string]interface{}{"sseType": v}
|
||||
}
|
||||
bucket.EncryptionConfiguration = cfg
|
||||
}
|
||||
writeJSON(w, http.StatusOK, &GetVectorBucketResponse{VectorBucket: bucket})
|
||||
}
|
||||
|
||||
func (s *Server) handleListVectorBuckets(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -547,3 +547,27 @@ func TestQueryVectorsTopKBounds(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestValidateEncryptionConfigurationRejectsCustomerKeys(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cfg map[string]interface{}
|
||||
want bool
|
||||
}{
|
||||
{"empty", nil, true},
|
||||
{"default sse", map[string]interface{}{"sseType": "AES256"}, true},
|
||||
{"tables wire name", map[string]interface{}{"sseAlgorithm": "AES256"}, false},
|
||||
{"both wire names", map[string]interface{}{"sseType": "AES256", "sseAlgorithm": "AES256"}, false},
|
||||
{"customer key", map[string]interface{}{"sseType": "aws:kms"}, false},
|
||||
{"tables kms", map[string]interface{}{"sseAlgorithm": "aws:kms"}, false},
|
||||
{"unknown sse", map[string]interface{}{"sseType": "aws:fsx"}, false},
|
||||
{"missing algorithm", map[string]interface{}{"other": "x"}, false},
|
||||
{"key without a type", map[string]interface{}{"kmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/x"}, false},
|
||||
{"key on default sse", map[string]interface{}{"sseType": "AES256", "kmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/x"}, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := s3tables.ValidateEncryptionConfiguration(tc.cfg, "sseType") == nil; got != tc.want {
|
||||
t.Fatalf("cfg %v: accepted=%v, want %v", tc.cfg, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -71,9 +71,10 @@ type GetVectorBucketResponse struct {
|
||||
}
|
||||
|
||||
type VectorBucket struct {
|
||||
VectorBucketArn string `json:"vectorBucketArn"`
|
||||
VectorBucketName string `json:"vectorBucketName"`
|
||||
CreationTime time.Time `json:"creationTime"`
|
||||
VectorBucketArn string `json:"vectorBucketArn"`
|
||||
VectorBucketName string `json:"vectorBucketName"`
|
||||
CreationTime time.Time `json:"creationTime"`
|
||||
EncryptionConfiguration map[string]interface{} `json:"encryptionConfiguration,omitempty"`
|
||||
}
|
||||
|
||||
type ListVectorBucketsRequest struct {
|
||||
|
||||
Reference in new issue
Block a user